1 use super::place
::PlaceRef
;
2 use super::{FunctionCx, LocalRef}
;
10 use rustc_middle
::mir
::interpret
::{ConstValue, Pointer, Scalar}
;
11 use rustc_middle
::ty
::layout
::{LayoutOf, TyAndLayout}
;
12 use rustc_middle
::ty
::Ty
;
13 use rustc_target
::abi
::{Abi, Align, Size}
;
17 /// The representation of a Rust value. The enum variant is in fact
18 /// uniquely determined by the value's type, but is kept as a
20 #[derive(Copy, Clone, Debug)]
21 pub enum OperandValue
<V
> {
22 /// A reference to the actual operand. The data is guaranteed
23 /// to be valid for the operand's lifetime.
24 /// The second value, if any, is the extra data (vtable or length)
25 /// which indicates that it refers to an unsized rvalue.
26 Ref(V
, Option
<V
>, Align
),
27 /// A single LLVM value.
29 /// A pair of immediate LLVM values. Used by fat pointers too.
33 /// An `OperandRef` is an "SSA" reference to a Rust value, along with
36 /// NOTE: unless you know a value's type exactly, you should not
37 /// generate LLVM opcodes acting on it and instead act via methods,
38 /// to avoid nasty edge cases. In particular, using `Builder::store`
39 /// directly is sure to cause problems -- use `OperandRef::store`
41 #[derive(Copy, Clone)]
42 pub struct OperandRef
<'tcx
, V
> {
44 pub val
: OperandValue
<V
>,
46 // The layout of value, based on its Rust type.
47 pub layout
: TyAndLayout
<'tcx
>,
50 impl<V
: CodegenObject
> fmt
::Debug
for OperandRef
<'_
, V
> {
51 fn fmt(&self, f
: &mut fmt
::Formatter
<'_
>) -> fmt
::Result
{
52 write
!(f
, "OperandRef({:?} @ {:?})", self.val
, self.layout
)
56 impl<'a
, 'tcx
, V
: CodegenObject
> OperandRef
<'tcx
, V
> {
57 pub fn new_zst
<Bx
: BuilderMethods
<'a
, 'tcx
, Value
= V
>>(
59 layout
: TyAndLayout
<'tcx
>,
60 ) -> OperandRef
<'tcx
, V
> {
61 assert
!(layout
.is_zst());
63 val
: OperandValue
::Immediate(bx
.const_undef(bx
.immediate_backend_type(layout
))),
68 pub fn from_const
<Bx
: BuilderMethods
<'a
, 'tcx
, Value
= V
>>(
70 val
: ConstValue
<'tcx
>,
73 let layout
= bx
.layout_of(ty
);
76 return OperandRef
::new_zst(bx
, layout
);
80 ConstValue
::Scalar(x
) => {
81 let Abi
::Scalar(scalar
) = layout
.abi
else {
82 bug
!("from_const: invalid ByVal layout: {:#?}", layout
);
84 let llval
= bx
.scalar_to_backend(x
, scalar
, bx
.immediate_backend_type(layout
));
85 OperandValue
::Immediate(llval
)
87 ConstValue
::Slice { data, start, end }
=> {
88 let Abi
::ScalarPair(a_scalar
, _
) = layout
.abi
else {
89 bug
!("from_const: invalid ScalarPair layout: {:#?}", layout
);
91 let a
= Scalar
::from_pointer(
92 Pointer
::new(bx
.tcx().create_memory_alloc(data
), Size
::from_bytes(start
)),
95 let a_llval
= bx
.scalar_to_backend(
98 bx
.scalar_pair_element_backend_type(layout
, 0, true),
100 let b_llval
= bx
.const_usize((end
- start
) as u64);
101 OperandValue
::Pair(a_llval
, b_llval
)
103 ConstValue
::ByRef { alloc, offset }
=> {
104 return bx
.load_operand(bx
.from_const_alloc(layout
, alloc
, offset
));
108 OperandRef { val, layout }
111 /// Asserts that this operand refers to a scalar and returns
112 /// a reference to its value.
113 pub fn immediate(self) -> V
{
115 OperandValue
::Immediate(s
) => s
,
116 _
=> bug
!("not immediate: {:?}", self),
120 pub fn deref
<Cx
: LayoutTypeMethods
<'tcx
>>(self, cx
: &Cx
) -> PlaceRef
<'tcx
, V
> {
121 let projected_ty
= self
125 .unwrap_or_else(|| bug
!("deref of non-pointer {:?}", self))
127 let (llptr
, llextra
) = match self.val
{
128 OperandValue
::Immediate(llptr
) => (llptr
, None
),
129 OperandValue
::Pair(llptr
, llextra
) => {
130 // if the box's allocator isn't a ZST, then "llextra" is actually the allocator
131 if self.layout
.ty
.is_box() && !self.layout
.field(cx
, 1).is_zst() {
134 (llptr
, Some(llextra
))
137 OperandValue
::Ref(..) => bug
!("Deref of by-Ref operand {:?}", self),
139 let layout
= cx
.layout_of(projected_ty
);
140 PlaceRef { llval: llptr, llextra, layout, align: layout.align.abi }
143 /// If this operand is a `Pair`, we return an aggregate with the two values.
144 /// For other cases, see `immediate`.
145 pub fn immediate_or_packed_pair
<Bx
: BuilderMethods
<'a
, 'tcx
, Value
= V
>>(
149 if let OperandValue
::Pair(a
, b
) = self.val
{
150 let llty
= bx
.cx().backend_type(self.layout
);
151 debug
!("Operand::immediate_or_packed_pair: packing {:?} into {:?}", self, llty
);
152 // Reconstruct the immediate aggregate.
153 let mut llpair
= bx
.cx().const_undef(llty
);
154 let imm_a
= bx
.from_immediate(a
);
155 let imm_b
= bx
.from_immediate(b
);
156 llpair
= bx
.insert_value(llpair
, imm_a
, 0);
157 llpair
= bx
.insert_value(llpair
, imm_b
, 1);
164 /// If the type is a pair, we return a `Pair`, otherwise, an `Immediate`.
165 pub fn from_immediate_or_packed_pair
<Bx
: BuilderMethods
<'a
, 'tcx
, Value
= V
>>(
168 layout
: TyAndLayout
<'tcx
>,
170 let val
= if let Abi
::ScalarPair(a
, b
) = layout
.abi
{
171 debug
!("Operand::from_immediate_or_packed_pair: unpacking {:?} @ {:?}", llval
, layout
);
173 // Deconstruct the immediate aggregate.
174 let a_llval
= bx
.extract_value(llval
, 0);
175 let a_llval
= bx
.to_immediate_scalar(a_llval
, a
);
176 let b_llval
= bx
.extract_value(llval
, 1);
177 let b_llval
= bx
.to_immediate_scalar(b_llval
, b
);
178 OperandValue
::Pair(a_llval
, b_llval
)
180 OperandValue
::Immediate(llval
)
182 OperandRef { val, layout }
185 pub fn extract_field
<Bx
: BuilderMethods
<'a
, 'tcx
, Value
= V
>>(
190 let field
= self.layout
.field(bx
.cx(), i
);
191 let offset
= self.layout
.fields
.offset(i
);
193 let mut val
= match (self.val
, self.layout
.abi
) {
194 // If the field is ZST, it has no data.
195 _
if field
.is_zst() => {
196 return OperandRef
::new_zst(bx
, field
);
199 // Newtype of a scalar, scalar pair or vector.
200 (OperandValue
::Immediate(_
) | OperandValue
::Pair(..), _
)
201 if field
.size
== self.layout
.size
=>
203 assert_eq
!(offset
.bytes(), 0);
207 // Extract a scalar component from a pair.
208 (OperandValue
::Pair(a_llval
, b_llval
), Abi
::ScalarPair(a
, b
)) => {
209 if offset
.bytes() == 0 {
210 assert_eq
!(field
.size
, a
.size(bx
.cx()));
211 OperandValue
::Immediate(a_llval
)
213 assert_eq
!(offset
, a
.size(bx
.cx()).align_to(b
.align(bx
.cx()).abi
));
214 assert_eq
!(field
.size
, b
.size(bx
.cx()));
215 OperandValue
::Immediate(b_llval
)
219 // `#[repr(simd)]` types are also immediate.
220 (OperandValue
::Immediate(llval
), Abi
::Vector { .. }
) => {
221 OperandValue
::Immediate(bx
.extract_element(llval
, bx
.cx().const_usize(i
as u64)))
224 _
=> bug
!("OperandRef::extract_field({:?}): not applicable", self),
227 match (&mut val
, field
.abi
) {
228 (OperandValue
::Immediate(llval
), _
) => {
229 // Bools in union fields needs to be truncated.
230 *llval
= bx
.to_immediate(*llval
, field
);
231 // HACK(eddyb) have to bitcast pointers until LLVM removes pointee types.
232 *llval
= bx
.bitcast(*llval
, bx
.cx().immediate_backend_type(field
));
234 (OperandValue
::Pair(a
, b
), Abi
::ScalarPair(a_abi
, b_abi
)) => {
235 // Bools in union fields needs to be truncated.
236 *a
= bx
.to_immediate_scalar(*a
, a_abi
);
237 *b
= bx
.to_immediate_scalar(*b
, b_abi
);
238 // HACK(eddyb) have to bitcast pointers until LLVM removes pointee types.
239 *a
= bx
.bitcast(*a
, bx
.cx().scalar_pair_element_backend_type(field
, 0, true));
240 *b
= bx
.bitcast(*b
, bx
.cx().scalar_pair_element_backend_type(field
, 1, true));
242 (OperandValue
::Pair(..), _
) => bug
!(),
243 (OperandValue
::Ref(..), _
) => bug
!(),
246 OperandRef { val, layout: field }
250 impl<'a
, 'tcx
, V
: CodegenObject
> OperandValue
<V
> {
251 pub fn store
<Bx
: BuilderMethods
<'a
, 'tcx
, Value
= V
>>(
254 dest
: PlaceRef
<'tcx
, V
>,
256 self.store_with_flags(bx
, dest
, MemFlags
::empty());
259 pub fn volatile_store
<Bx
: BuilderMethods
<'a
, 'tcx
, Value
= V
>>(
262 dest
: PlaceRef
<'tcx
, V
>,
264 self.store_with_flags(bx
, dest
, MemFlags
::VOLATILE
);
267 pub fn unaligned_volatile_store
<Bx
: BuilderMethods
<'a
, 'tcx
, Value
= V
>>(
270 dest
: PlaceRef
<'tcx
, V
>,
272 self.store_with_flags(bx
, dest
, MemFlags
::VOLATILE
| MemFlags
::UNALIGNED
);
275 pub fn nontemporal_store
<Bx
: BuilderMethods
<'a
, 'tcx
, Value
= V
>>(
278 dest
: PlaceRef
<'tcx
, V
>,
280 self.store_with_flags(bx
, dest
, MemFlags
::NONTEMPORAL
);
283 fn store_with_flags
<Bx
: BuilderMethods
<'a
, 'tcx
, Value
= V
>>(
286 dest
: PlaceRef
<'tcx
, V
>,
289 debug
!("OperandRef::store: operand={:?}, dest={:?}", self, dest
);
290 // Avoid generating stores of zero-sized values, because the only way to have a zero-sized
291 // value is through `undef`, and store itself is useless.
292 if dest
.layout
.is_zst() {
296 OperandValue
::Ref(r
, None
, source_align
) => {
297 if flags
.contains(MemFlags
::NONTEMPORAL
) {
298 // HACK(nox): This is inefficient but there is no nontemporal memcpy.
299 let ty
= bx
.backend_type(dest
.layout
);
300 let ptr
= bx
.pointercast(r
, bx
.type_ptr_to(ty
));
301 let val
= bx
.load(ty
, ptr
, source_align
);
302 bx
.store_with_flags(val
, dest
.llval
, dest
.align
, flags
);
305 base
::memcpy_ty(bx
, dest
.llval
, dest
.align
, r
, source_align
, dest
.layout
, flags
)
307 OperandValue
::Ref(_
, Some(_
), _
) => {
308 bug
!("cannot directly store unsized values");
310 OperandValue
::Immediate(s
) => {
311 let val
= bx
.from_immediate(s
);
312 bx
.store_with_flags(val
, dest
.llval
, dest
.align
, flags
);
314 OperandValue
::Pair(a
, b
) => {
315 let Abi
::ScalarPair(a_scalar
, b_scalar
) = dest
.layout
.abi
else {
316 bug
!("store_with_flags: invalid ScalarPair layout: {:#?}", dest
.layout
);
318 let ty
= bx
.backend_type(dest
.layout
);
319 let b_offset
= a_scalar
.size(bx
).align_to(b_scalar
.align(bx
).abi
);
321 let llptr
= bx
.struct_gep(ty
, dest
.llval
, 0);
322 let val
= bx
.from_immediate(a
);
323 let align
= dest
.align
;
324 bx
.store_with_flags(val
, llptr
, align
, flags
);
326 let llptr
= bx
.struct_gep(ty
, dest
.llval
, 1);
327 let val
= bx
.from_immediate(b
);
328 let align
= dest
.align
.restrict_for_offset(b_offset
);
329 bx
.store_with_flags(val
, llptr
, align
, flags
);
334 pub fn store_unsized
<Bx
: BuilderMethods
<'a
, 'tcx
, Value
= V
>>(
337 indirect_dest
: PlaceRef
<'tcx
, V
>,
339 debug
!("OperandRef::store_unsized: operand={:?}, indirect_dest={:?}", self, indirect_dest
);
340 let flags
= MemFlags
::empty();
342 // `indirect_dest` must have `*mut T` type. We extract `T` out of it.
343 let unsized_ty
= indirect_dest
347 .unwrap_or_else(|| bug
!("indirect_dest has non-pointer type: {:?}", indirect_dest
))
350 let OperandValue
::Ref(llptr
, Some(llextra
), _
) = self else {
351 bug
!("store_unsized called with a sized value")
354 // FIXME: choose an appropriate alignment, or use dynamic align somehow
355 let max_align
= Align
::from_bits(128).unwrap();
356 let min_align
= Align
::from_bits(8).unwrap();
358 // Allocate an appropriate region on the stack, and copy the value into it
359 let (llsize
, _
) = glue
::size_and_align_of_dst(bx
, unsized_ty
, Some(llextra
));
360 let lldst
= bx
.array_alloca(bx
.cx().type_i8(), llsize
, max_align
);
361 bx
.memcpy(lldst
, max_align
, llptr
, min_align
, llsize
, flags
);
363 // Store the allocated region and the extra to the indirect place.
364 let indirect_operand
= OperandValue
::Pair(lldst
, llextra
);
365 indirect_operand
.store(bx
, indirect_dest
);
369 impl<'a
, 'tcx
, Bx
: BuilderMethods
<'a
, 'tcx
>> FunctionCx
<'a
, 'tcx
, Bx
> {
370 fn maybe_codegen_consume_direct(
373 place_ref
: mir
::PlaceRef
<'tcx
>,
374 ) -> Option
<OperandRef
<'tcx
, Bx
::Value
>> {
375 debug
!("maybe_codegen_consume_direct(place_ref={:?})", place_ref
);
377 match self.locals
[place_ref
.local
] {
378 LocalRef
::Operand(Some(mut o
)) => {
379 // Moves out of scalar and scalar pair fields are trivial.
380 for elem
in place_ref
.projection
.iter() {
382 mir
::ProjectionElem
::Field(ref f
, _
) => {
383 o
= o
.extract_field(bx
, f
.index());
385 mir
::ProjectionElem
::Index(_
)
386 | mir
::ProjectionElem
::ConstantIndex { .. }
=> {
387 // ZSTs don't require any actual memory access.
388 // FIXME(eddyb) deduplicate this with the identical
389 // checks in `codegen_consume` and `extract_field`.
390 let elem
= o
.layout
.field(bx
.cx(), 0);
392 o
= OperandRef
::new_zst(bx
, elem
);
403 LocalRef
::Operand(None
) => {
404 bug
!("use of {:?} before def", place_ref
);
406 LocalRef
::Place(..) | LocalRef
::UnsizedPlace(..) => {
407 // watch out for locals that do not have an
408 // alloca; they are handled somewhat differently
414 pub fn codegen_consume(
417 place_ref
: mir
::PlaceRef
<'tcx
>,
418 ) -> OperandRef
<'tcx
, Bx
::Value
> {
419 debug
!("codegen_consume(place_ref={:?})", place_ref
);
421 let ty
= self.monomorphized_place_ty(place_ref
);
422 let layout
= bx
.cx().layout_of(ty
);
424 // ZSTs don't require any actual memory access.
426 return OperandRef
::new_zst(bx
, layout
);
429 if let Some(o
) = self.maybe_codegen_consume_direct(bx
, place_ref
) {
433 // for most places, to consume them we just load them
434 // out from their home
435 let place
= self.codegen_place(bx
, place_ref
);
436 bx
.load_operand(place
)
439 pub fn codegen_operand(
442 operand
: &mir
::Operand
<'tcx
>,
443 ) -> OperandRef
<'tcx
, Bx
::Value
> {
444 debug
!("codegen_operand(operand={:?})", operand
);
447 mir
::Operand
::Copy(ref place
) | mir
::Operand
::Move(ref place
) => {
448 self.codegen_consume(bx
, place
.as_ref())
451 mir
::Operand
::Constant(ref constant
) => {
452 // This cannot fail because we checked all required_consts in advance.
453 self.eval_mir_constant_to_operand(bx
, constant
).unwrap_or_else(|_err
| {
454 span_bug
!(constant
.span
, "erroneous constant not captured by required_consts")