1 use super::operand
::OperandRef
;
2 use super::operand
::OperandValue
::{Immediate, Pair, Ref}
;
3 use super::place
::PlaceRef
;
4 use super::{FunctionCx, LocalRef}
;
7 use crate::common
::{self, IntPredicate}
;
13 use rustc_hir
::lang_items
::LangItem
;
14 use rustc_index
::vec
::Idx
;
15 use rustc_middle
::mir
::interpret
::ConstValue
;
16 use rustc_middle
::mir
::AssertKind
;
17 use rustc_middle
::mir
::{self, SwitchTargets}
;
18 use rustc_middle
::ty
::layout
::{FnAbiExt, HasTyCtxt}
;
19 use rustc_middle
::ty
::print
::with_no_trimmed_paths
;
20 use rustc_middle
::ty
::{self, Instance, Ty, TypeFoldable}
;
21 use rustc_span
::source_map
::Span
;
22 use rustc_span
::{sym, Symbol}
;
23 use rustc_target
::abi
::call
::{ArgAbi, FnAbi, PassMode}
;
24 use rustc_target
::abi
::{self, LayoutOf}
;
25 use rustc_target
::spec
::abi
::Abi
;
27 /// Used by `FunctionCx::codegen_terminator` for emitting common patterns
28 /// e.g., creating a basic block, calling a function, etc.
29 struct TerminatorCodegenHelper
<'tcx
> {
31 terminator
: &'tcx mir
::Terminator
<'tcx
>,
32 funclet_bb
: Option
<mir
::BasicBlock
>,
35 impl<'a
, 'tcx
> TerminatorCodegenHelper
<'tcx
> {
36 /// Returns the associated funclet from `FunctionCx::funclets` for the
37 /// `funclet_bb` member if it is not `None`.
38 fn funclet
<'b
, Bx
: BuilderMethods
<'a
, 'tcx
>>(
40 fx
: &'b FunctionCx
<'a
, 'tcx
, Bx
>,
41 ) -> Option
<&'b Bx
::Funclet
> {
42 self.funclet_bb
.and_then(|funcl
| fx
.funclets
[funcl
].as_ref())
45 fn lltarget
<Bx
: BuilderMethods
<'a
, 'tcx
>>(
47 fx
: &mut FunctionCx
<'a
, 'tcx
, Bx
>,
48 target
: mir
::BasicBlock
,
49 ) -> (Bx
::BasicBlock
, bool
) {
50 let span
= self.terminator
.source_info
.span
;
51 let lltarget
= fx
.blocks
[target
];
52 let target_funclet
= fx
.cleanup_kinds
[target
].funclet_bb(target
);
53 match (self.funclet_bb
, target_funclet
) {
54 (None
, None
) => (lltarget
, false),
55 (Some(f
), Some(t_f
)) if f
== t_f
|| !base
::wants_msvc_seh(fx
.cx
.tcx().sess
) => {
58 // jump *into* cleanup - need a landing pad if GNU
59 (None
, Some(_
)) => (fx
.landing_pad_to(target
), false),
60 (Some(_
), None
) => span_bug
!(span
, "{:?} - jump out of cleanup?", self.terminator
),
61 (Some(_
), Some(_
)) => (fx
.landing_pad_to(target
), true),
65 /// Create a basic block.
66 fn llblock
<Bx
: BuilderMethods
<'a
, 'tcx
>>(
68 fx
: &mut FunctionCx
<'a
, 'tcx
, Bx
>,
69 target
: mir
::BasicBlock
,
71 let (lltarget
, is_cleanupret
) = self.lltarget(fx
, target
);
73 // MSVC cross-funclet jump - need a trampoline
75 debug
!("llblock: creating cleanup trampoline for {:?}", target
);
76 let name
= &format
!("{:?}_cleanup_trampoline_{:?}", self.bb
, target
);
77 let mut trampoline
= fx
.new_block(name
);
78 trampoline
.cleanup_ret(self.funclet(fx
).unwrap(), Some(lltarget
));
85 fn funclet_br
<Bx
: BuilderMethods
<'a
, 'tcx
>>(
87 fx
: &mut FunctionCx
<'a
, 'tcx
, Bx
>,
89 target
: mir
::BasicBlock
,
91 let (lltarget
, is_cleanupret
) = self.lltarget(fx
, target
);
93 // micro-optimization: generate a `ret` rather than a jump
95 bx
.cleanup_ret(self.funclet(fx
).unwrap(), Some(lltarget
));
101 /// Call `fn_ptr` of `fn_abi` with the arguments `llargs`, the optional
102 /// return destination `destination` and the cleanup function `cleanup`.
103 fn do_call
<Bx
: BuilderMethods
<'a
, 'tcx
>>(
105 fx
: &mut FunctionCx
<'a
, 'tcx
, Bx
>,
107 fn_abi
: FnAbi
<'tcx
, Ty
<'tcx
>>,
109 llargs
: &[Bx
::Value
],
110 destination
: Option
<(ReturnDest
<'tcx
, Bx
::Value
>, mir
::BasicBlock
)>,
111 cleanup
: Option
<mir
::BasicBlock
>,
113 // If there is a cleanup block and the function we're calling can unwind, then
114 // do an invoke, otherwise do a call.
115 if let Some(cleanup
) = cleanup
.filter(|_
| fn_abi
.can_unwind
) {
116 let ret_bx
= if let Some((_
, target
)) = destination
{
119 fx
.unreachable_block()
122 bx
.invoke(fn_ptr
, &llargs
, ret_bx
, self.llblock(fx
, cleanup
), self.funclet(fx
));
123 bx
.apply_attrs_callsite(&fn_abi
, invokeret
);
125 if let Some((ret_dest
, target
)) = destination
{
126 let mut ret_bx
= fx
.build_block(target
);
127 fx
.set_debug_loc(&mut ret_bx
, self.terminator
.source_info
);
128 fx
.store_return(&mut ret_bx
, ret_dest
, &fn_abi
.ret
, invokeret
);
131 let llret
= bx
.call(fn_ptr
, &llargs
, self.funclet(fx
));
132 bx
.apply_attrs_callsite(&fn_abi
, llret
);
133 if fx
.mir
[self.bb
].is_cleanup
{
134 // Cleanup is always the cold path. Don't inline
135 // drop glue. Also, when there is a deeply-nested
136 // struct, there are "symmetry" issues that cause
137 // exponential inlining - see issue #41696.
138 bx
.do_not_inline(llret
);
141 if let Some((ret_dest
, target
)) = destination
{
142 fx
.store_return(bx
, ret_dest
, &fn_abi
.ret
, llret
);
143 self.funclet_br(fx
, bx
, target
);
151 /// Codegen implementations for some terminator variants.
152 impl<'a
, 'tcx
, Bx
: BuilderMethods
<'a
, 'tcx
>> FunctionCx
<'a
, 'tcx
, Bx
> {
153 /// Generates code for a `Resume` terminator.
154 fn codegen_resume_terminator(&mut self, helper
: TerminatorCodegenHelper
<'tcx
>, mut bx
: Bx
) {
155 if let Some(funclet
) = helper
.funclet(self) {
156 bx
.cleanup_ret(funclet
, None
);
158 let slot
= self.get_personality_slot(&mut bx
);
159 let lp0
= slot
.project_field(&mut bx
, 0);
160 let lp0
= bx
.load_operand(lp0
).immediate();
161 let lp1
= slot
.project_field(&mut bx
, 1);
162 let lp1
= bx
.load_operand(lp1
).immediate();
163 slot
.storage_dead(&mut bx
);
165 let mut lp
= bx
.const_undef(self.landing_pad_type());
166 lp
= bx
.insert_value(lp
, lp0
, 0);
167 lp
= bx
.insert_value(lp
, lp1
, 1);
172 fn codegen_switchint_terminator(
174 helper
: TerminatorCodegenHelper
<'tcx
>,
176 discr
: &mir
::Operand
<'tcx
>,
178 targets
: &SwitchTargets
,
180 let discr
= self.codegen_operand(&mut bx
, &discr
);
181 // `switch_ty` is redundant, sanity-check that.
182 assert_eq
!(discr
.layout
.ty
, switch_ty
);
183 let mut target_iter
= targets
.iter();
184 if target_iter
.len() == 1 {
185 // If there are two targets (one conditional, one fallback), emit br instead of switch
186 let (test_value
, target
) = target_iter
.next().unwrap();
187 let lltrue
= helper
.llblock(self, target
);
188 let llfalse
= helper
.llblock(self, targets
.otherwise());
189 if switch_ty
== bx
.tcx().types
.bool
{
190 // Don't generate trivial icmps when switching on bool
192 0 => bx
.cond_br(discr
.immediate(), llfalse
, lltrue
),
193 1 => bx
.cond_br(discr
.immediate(), lltrue
, llfalse
),
197 let switch_llty
= bx
.immediate_backend_type(bx
.layout_of(switch_ty
));
198 let llval
= bx
.const_uint_big(switch_llty
, test_value
);
199 let cmp
= bx
.icmp(IntPredicate
::IntEQ
, discr
.immediate(), llval
);
200 bx
.cond_br(cmp
, lltrue
, llfalse
);
205 helper
.llblock(self, targets
.otherwise()),
206 target_iter
.map(|(value
, target
)| (value
, helper
.llblock(self, target
))),
211 fn codegen_return_terminator(&mut self, mut bx
: Bx
) {
212 // Call `va_end` if this is the definition of a C-variadic function.
213 if self.fn_abi
.c_variadic
{
214 // The `VaList` "spoofed" argument is just after all the real arguments.
215 let va_list_arg_idx
= self.fn_abi
.args
.len();
216 match self.locals
[mir
::Local
::new(1 + va_list_arg_idx
)] {
217 LocalRef
::Place(va_list
) => {
218 bx
.va_end(va_list
.llval
);
220 _
=> bug
!("C-variadic function must have a `VaList` place"),
223 if self.fn_abi
.ret
.layout
.abi
.is_uninhabited() {
224 // Functions with uninhabited return values are marked `noreturn`,
225 // so we should make sure that we never actually do.
226 // We play it safe by using a well-defined `abort`, but we could go for immediate UB
227 // if that turns out to be helpful.
229 // `abort` does not terminate the block, so we still need to generate
230 // an `unreachable` terminator after it.
234 let llval
= match self.fn_abi
.ret
.mode
{
235 PassMode
::Ignore
| PassMode
::Indirect { .. }
=> {
240 PassMode
::Direct(_
) | PassMode
::Pair(..) => {
241 let op
= self.codegen_consume(&mut bx
, mir
::Place
::return_place().as_ref());
242 if let Ref(llval
, _
, align
) = op
.val
{
243 bx
.load(llval
, align
)
245 op
.immediate_or_packed_pair(&mut bx
)
249 PassMode
::Cast(cast_ty
) => {
250 let op
= match self.locals
[mir
::RETURN_PLACE
] {
251 LocalRef
::Operand(Some(op
)) => op
,
252 LocalRef
::Operand(None
) => bug
!("use of return before def"),
253 LocalRef
::Place(cg_place
) => OperandRef
{
254 val
: Ref(cg_place
.llval
, None
, cg_place
.align
),
255 layout
: cg_place
.layout
,
257 LocalRef
::UnsizedPlace(_
) => bug
!("return type must be sized"),
259 let llslot
= match op
.val
{
260 Immediate(_
) | Pair(..) => {
261 let scratch
= PlaceRef
::alloca(&mut bx
, self.fn_abi
.ret
.layout
);
262 op
.val
.store(&mut bx
, scratch
);
265 Ref(llval
, _
, align
) => {
266 assert_eq
!(align
, op
.layout
.align
.abi
, "return place is unaligned!");
270 let addr
= bx
.pointercast(llslot
, bx
.type_ptr_to(bx
.cast_backend_type(&cast_ty
)));
271 bx
.load(addr
, self.fn_abi
.ret
.layout
.align
.abi
)
277 fn codegen_drop_terminator(
279 helper
: TerminatorCodegenHelper
<'tcx
>,
281 location
: mir
::Place
<'tcx
>,
282 target
: mir
::BasicBlock
,
283 unwind
: Option
<mir
::BasicBlock
>,
285 let ty
= location
.ty(self.mir
, bx
.tcx()).ty
;
286 let ty
= self.monomorphize(ty
);
287 let drop_fn
= Instance
::resolve_drop_in_place(bx
.tcx(), ty
);
289 if let ty
::InstanceDef
::DropGlue(_
, None
) = drop_fn
.def
{
290 // we don't actually need to drop anything.
291 helper
.funclet_br(self, &mut bx
, target
);
295 let place
= self.codegen_place(&mut bx
, location
.as_ref());
297 let mut args
= if let Some(llextra
) = place
.llextra
{
298 args2
= [place
.llval
, llextra
];
301 args1
= [place
.llval
];
304 let (drop_fn
, fn_abi
) = match ty
.kind() {
305 // FIXME(eddyb) perhaps move some of this logic into
306 // `Instance::resolve_drop_in_place`?
308 let virtual_drop
= Instance
{
309 def
: ty
::InstanceDef
::Virtual(drop_fn
.def_id(), 0),
310 substs
: drop_fn
.substs
,
312 let fn_abi
= FnAbi
::of_instance(&bx
, virtual_drop
, &[]);
313 let vtable
= args
[1];
315 (meth
::DESTRUCTOR
.get_fn(&mut bx
, vtable
, &fn_abi
), fn_abi
)
317 _
=> (bx
.get_fn_addr(drop_fn
), FnAbi
::of_instance(&bx
, drop_fn
, &[])),
325 Some((ReturnDest
::Nothing
, target
)),
330 fn codegen_assert_terminator(
332 helper
: TerminatorCodegenHelper
<'tcx
>,
334 terminator
: &mir
::Terminator
<'tcx
>,
335 cond
: &mir
::Operand
<'tcx
>,
337 msg
: &mir
::AssertMessage
<'tcx
>,
338 target
: mir
::BasicBlock
,
339 cleanup
: Option
<mir
::BasicBlock
>,
341 let span
= terminator
.source_info
.span
;
342 let cond
= self.codegen_operand(&mut bx
, cond
).immediate();
343 let mut const_cond
= bx
.const_to_opt_u128(cond
, false).map(|c
| c
== 1);
345 // This case can currently arise only from functions marked
346 // with #[rustc_inherit_overflow_checks] and inlined from
347 // another crate (mostly core::num generic/#[inline] fns),
348 // while the current crate doesn't use overflow checks.
349 // NOTE: Unlike binops, negation doesn't have its own
350 // checked operation, just a comparison with the minimum
351 // value, so we have to check for the assert message.
352 if !bx
.check_overflow() {
353 if let AssertKind
::OverflowNeg(_
) = *msg
{
354 const_cond
= Some(expected
);
358 // Don't codegen the panic block if success if known.
359 if const_cond
== Some(expected
) {
360 helper
.funclet_br(self, &mut bx
, target
);
364 // Pass the condition through llvm.expect for branch hinting.
365 let cond
= bx
.expect(cond
, expected
);
367 // Create the failure block and the conditional branch to it.
368 let lltarget
= helper
.llblock(self, target
);
369 let panic_block
= self.new_block("panic");
371 bx
.cond_br(cond
, lltarget
, panic_block
.llbb());
373 bx
.cond_br(cond
, panic_block
.llbb(), lltarget
);
376 // After this point, bx is the block for the call to panic.
378 self.set_debug_loc(&mut bx
, terminator
.source_info
);
380 // Get the location information.
381 let location
= self.get_caller_location(&mut bx
, terminator
.source_info
).immediate();
383 // Put together the arguments to the panic entry point.
384 let (lang_item
, args
) = match msg
{
385 AssertKind
::BoundsCheck { ref len, ref index }
=> {
386 let len
= self.codegen_operand(&mut bx
, len
).immediate();
387 let index
= self.codegen_operand(&mut bx
, index
).immediate();
388 // It's `fn panic_bounds_check(index: usize, len: usize)`,
389 // and `#[track_caller]` adds an implicit third argument.
390 (LangItem
::PanicBoundsCheck
, vec
![index
, len
, location
])
393 let msg_str
= Symbol
::intern(msg
.description());
394 let msg
= bx
.const_str(msg_str
);
395 // It's `pub fn panic(expr: &str)`, with the wide reference being passed
396 // as two arguments, and `#[track_caller]` adds an implicit third argument.
397 (LangItem
::Panic
, vec
![msg
.0, msg
.1, location
])
401 // Obtain the panic entry point.
402 let def_id
= common
::langcall(bx
.tcx(), Some(span
), "", lang_item
);
403 let instance
= ty
::Instance
::mono(bx
.tcx(), def_id
);
404 let fn_abi
= FnAbi
::of_instance(&bx
, instance
, &[]);
405 let llfn
= bx
.get_fn_addr(instance
);
407 // Codegen the actual panic invoke/call.
408 helper
.do_call(self, &mut bx
, fn_abi
, llfn
, &args
, None
, cleanup
);
411 /// Returns `true` if this is indeed a panic intrinsic and codegen is done.
412 fn codegen_panic_intrinsic(
414 helper
: &TerminatorCodegenHelper
<'tcx
>,
416 intrinsic
: Option
<Symbol
>,
417 instance
: Option
<Instance
<'tcx
>>,
418 source_info
: mir
::SourceInfo
,
419 destination
: &Option
<(mir
::Place
<'tcx
>, mir
::BasicBlock
)>,
420 cleanup
: Option
<mir
::BasicBlock
>,
422 // Emit a panic or a no-op for `assert_*` intrinsics.
423 // These are intrinsics that compile to panics so that we can get a message
424 // which mentions the offending type, even from a const context.
425 #[derive(Debug, PartialEq)]
426 enum AssertIntrinsic
{
431 let panic_intrinsic
= intrinsic
.and_then(|i
| match i
{
432 sym
::assert_inhabited
=> Some(AssertIntrinsic
::Inhabited
),
433 sym
::assert_zero_valid
=> Some(AssertIntrinsic
::ZeroValid
),
434 sym
::assert_uninit_valid
=> Some(AssertIntrinsic
::UninitValid
),
437 if let Some(intrinsic
) = panic_intrinsic
{
438 use AssertIntrinsic
::*;
439 let ty
= instance
.unwrap().substs
.type_at(0);
440 let layout
= bx
.layout_of(ty
);
441 let do_panic
= match intrinsic
{
442 Inhabited
=> layout
.abi
.is_uninhabited(),
443 // We unwrap as the error type is `!`.
444 ZeroValid
=> !layout
.might_permit_raw_init(bx
, /*zero:*/ true).unwrap(),
445 // We unwrap as the error type is `!`.
446 UninitValid
=> !layout
.might_permit_raw_init(bx
, /*zero:*/ false).unwrap(),
449 let msg_str
= with_no_trimmed_paths(|| {
450 if layout
.abi
.is_uninhabited() {
451 // Use this error even for the other intrinsics as it is more precise.
452 format
!("attempted to instantiate uninhabited type `{}`", ty
)
453 } else if intrinsic
== ZeroValid
{
454 format
!("attempted to zero-initialize type `{}`, which is invalid", ty
)
456 format
!("attempted to leave type `{}` uninitialized, which is invalid", ty
)
459 let msg
= bx
.const_str(Symbol
::intern(&msg_str
));
460 let location
= self.get_caller_location(bx
, source_info
).immediate();
462 // Obtain the panic entry point.
463 // FIXME: dedup this with `codegen_assert_terminator` above.
465 common
::langcall(bx
.tcx(), Some(source_info
.span
), "", LangItem
::Panic
);
466 let instance
= ty
::Instance
::mono(bx
.tcx(), def_id
);
467 let fn_abi
= FnAbi
::of_instance(bx
, instance
, &[]);
468 let llfn
= bx
.get_fn_addr(instance
);
470 // Codegen the actual panic invoke/call.
476 &[msg
.0, msg
.1, location
],
477 destination
.as_ref().map(|(_
, bb
)| (ReturnDest
::Nothing
, *bb
)),
482 let target
= destination
.as_ref().unwrap().1;
483 helper
.funclet_br(self, bx
, target
)
491 fn codegen_call_terminator(
493 helper
: TerminatorCodegenHelper
<'tcx
>,
495 terminator
: &mir
::Terminator
<'tcx
>,
496 func
: &mir
::Operand
<'tcx
>,
497 args
: &[mir
::Operand
<'tcx
>],
498 destination
: &Option
<(mir
::Place
<'tcx
>, mir
::BasicBlock
)>,
499 cleanup
: Option
<mir
::BasicBlock
>,
502 let source_info
= terminator
.source_info
;
503 let span
= source_info
.span
;
505 // Create the callee. This is a fn ptr or zero-sized and hence a kind of scalar.
506 let callee
= self.codegen_operand(&mut bx
, func
);
508 let (instance
, mut llfn
) = match *callee
.layout
.ty
.kind() {
509 ty
::FnDef(def_id
, substs
) => (
511 ty
::Instance
::resolve(bx
.tcx(), ty
::ParamEnv
::reveal_all(), def_id
, substs
)
514 .polymorphize(bx
.tcx()),
518 ty
::FnPtr(_
) => (None
, Some(callee
.immediate())),
519 _
=> bug
!("{} is not callable", callee
.layout
.ty
),
521 let def
= instance
.map(|i
| i
.def
);
523 if let Some(ty
::InstanceDef
::DropGlue(_
, None
)) = def
{
524 // Empty drop glue; a no-op.
525 let &(_
, target
) = destination
.as_ref().unwrap();
526 helper
.funclet_br(self, &mut bx
, target
);
530 // FIXME(eddyb) avoid computing this if possible, when `instance` is
531 // available - right now `sig` is only needed for getting the `abi`
532 // and figuring out how many extra args were passed to a C-variadic `fn`.
533 let sig
= callee
.layout
.ty
.fn_sig(bx
.tcx());
536 // Handle intrinsics old codegen wants Expr's for, ourselves.
537 let intrinsic
= match def
{
538 Some(ty
::InstanceDef
::Intrinsic(def_id
)) => Some(bx
.tcx().item_name(def_id
)),
542 let extra_args
= &args
[sig
.inputs().skip_binder().len()..];
543 let extra_args
= extra_args
546 let op_ty
= op_arg
.ty(self.mir
, bx
.tcx());
547 self.monomorphize(op_ty
)
549 .collect
::<Vec
<_
>>();
551 let fn_abi
= match instance
{
552 Some(instance
) => FnAbi
::of_instance(&bx
, instance
, &extra_args
),
553 None
=> FnAbi
::of_fn_ptr(&bx
, sig
, &extra_args
),
556 if intrinsic
== Some(sym
::transmute
) {
557 if let Some(destination_ref
) = destination
.as_ref() {
558 let &(dest
, target
) = destination_ref
;
559 self.codegen_transmute(&mut bx
, &args
[0], dest
);
560 helper
.funclet_br(self, &mut bx
, target
);
562 // If we are trying to transmute to an uninhabited type,
563 // it is likely there is no allotted destination. In fact,
564 // transmuting to an uninhabited type is UB, which means
565 // we can do what we like. Here, we declare that transmuting
566 // into an uninhabited type is impossible, so anything following
567 // it must be unreachable.
568 assert_eq
!(fn_abi
.ret
.layout
.abi
, abi
::Abi
::Uninhabited
);
574 if self.codegen_panic_intrinsic(
586 // The arguments we'll be passing. Plus one to account for outptr, if used.
587 let arg_count
= fn_abi
.args
.len() + fn_abi
.ret
.is_indirect() as usize;
588 let mut llargs
= Vec
::with_capacity(arg_count
);
590 // Prepare the return value destination
591 let ret_dest
= if let Some((dest
, _
)) = *destination
{
592 let is_intrinsic
= intrinsic
.is_some();
593 self.make_return_dest(&mut bx
, dest
, &fn_abi
.ret
, &mut llargs
, is_intrinsic
)
598 if intrinsic
== Some(sym
::caller_location
) {
599 if let Some((_
, target
)) = destination
.as_ref() {
601 .get_caller_location(&mut bx
, mir
::SourceInfo { span: fn_span, ..source_info }
);
603 if let ReturnDest
::IndirectOperand(tmp
, _
) = ret_dest
{
604 location
.val
.store(&mut bx
, tmp
);
606 self.store_return(&mut bx
, ret_dest
, &fn_abi
.ret
, location
.immediate());
607 helper
.funclet_br(self, &mut bx
, *target
);
613 None
| Some(sym
::drop_in_place
) => {}
614 Some(sym
::copy_nonoverlapping
) => unreachable
!(),
616 let dest
= match ret_dest
{
617 _
if fn_abi
.ret
.is_indirect() => llargs
[0],
618 ReturnDest
::Nothing
=> {
619 bx
.const_undef(bx
.type_ptr_to(bx
.arg_memory_ty(&fn_abi
.ret
)))
621 ReturnDest
::IndirectOperand(dst
, _
) | ReturnDest
::Store(dst
) => dst
.llval
,
622 ReturnDest
::DirectOperand(_
) => {
623 bug
!("Cannot use direct operand with an intrinsic call")
627 let args
: Vec
<_
> = args
631 // The indices passed to simd_shuffle* in the
632 // third argument must be constant. This is
633 // checked by const-qualification, which also
634 // promotes any complex rvalues to constants.
635 if i
== 2 && intrinsic
.as_str().starts_with("simd_shuffle") {
636 if let mir
::Operand
::Constant(constant
) = arg
{
637 let c
= self.eval_mir_constant(constant
);
639 self.simd_shuffle_indices(&bx
, constant
.span
, constant
.ty(), c
);
641 val
: Immediate(llval
),
642 layout
: bx
.layout_of(ty
),
645 span_bug
!(span
, "shuffle indices must be constant");
649 self.codegen_operand(&mut bx
, arg
)
653 Self::codegen_intrinsic_call(
655 *instance
.as_ref().unwrap(),
662 if let ReturnDest
::IndirectOperand(dst
, _
) = ret_dest
{
663 self.store_return(&mut bx
, ret_dest
, &fn_abi
.ret
, dst
.llval
);
666 if let Some((_
, target
)) = *destination
{
667 helper
.funclet_br(self, &mut bx
, target
);
676 // Split the rust-call tupled arguments off.
677 let (first_args
, untuple
) = if abi
== Abi
::RustCall
&& !args
.is_empty() {
678 let (tup
, args
) = args
.split_last().unwrap();
684 'make_args
: for (i
, arg
) in first_args
.iter().enumerate() {
685 let mut op
= self.codegen_operand(&mut bx
, arg
);
687 if let (0, Some(ty
::InstanceDef
::Virtual(_
, idx
))) = (i
, def
) {
688 if let Pair(..) = op
.val
{
689 // In the case of Rc<Self>, we need to explicitly pass a
690 // *mut RcBox<Self> with a Scalar (not ScalarPair) ABI. This is a hack
691 // that is understood elsewhere in the compiler as a method on
693 // To get a `*mut RcBox<Self>`, we just keep unwrapping newtypes until
694 // we get a value of a built-in pointer type
695 'descend_newtypes
: while !op
.layout
.ty
.is_unsafe_ptr()
696 && !op
.layout
.ty
.is_region_ptr()
698 for i
in 0..op
.layout
.fields
.count() {
699 let field
= op
.extract_field(&mut bx
, i
);
700 if !field
.layout
.is_zst() {
701 // we found the one non-zero-sized field that is allowed
702 // now find *its* non-zero-sized field, or stop if it's a
705 continue 'descend_newtypes
;
709 span_bug
!(span
, "receiver has no non-zero-sized fields {:?}", op
);
712 // now that we have `*dyn Trait` or `&dyn Trait`, split it up into its
713 // data pointer and vtable. Look up the method in the vtable, and pass
714 // the data pointer as the first argument
716 Pair(data_ptr
, meta
) => {
718 meth
::VirtualIndex
::from_index(idx
).get_fn(&mut bx
, meta
, &fn_abi
),
720 llargs
.push(data_ptr
);
723 other
=> bug
!("expected a Pair, got {:?}", other
),
725 } else if let Ref(data_ptr
, Some(meta
), _
) = op
.val
{
726 // by-value dynamic dispatch
727 llfn
= Some(meth
::VirtualIndex
::from_index(idx
).get_fn(&mut bx
, meta
, &fn_abi
));
728 llargs
.push(data_ptr
);
731 span_bug
!(span
, "can't codegen a virtual call on {:?}", op
);
735 // The callee needs to own the argument memory if we pass it
736 // by-ref, so make a local copy of non-immediate constants.
737 match (arg
, op
.val
) {
738 (&mir
::Operand
::Copy(_
), Ref(_
, None
, _
))
739 | (&mir
::Operand
::Constant(_
), Ref(_
, None
, _
)) => {
740 let tmp
= PlaceRef
::alloca(&mut bx
, op
.layout
);
741 op
.val
.store(&mut bx
, tmp
);
742 op
.val
= Ref(tmp
.llval
, None
, tmp
.align
);
747 self.codegen_argument(&mut bx
, op
, &mut llargs
, &fn_abi
.args
[i
]);
749 if let Some(tup
) = untuple
{
750 self.codegen_arguments_untupled(
754 &fn_abi
.args
[first_args
.len()..],
759 instance
.map_or(false, |i
| i
.def
.requires_caller_location(self.cx
.tcx()));
764 "#[track_caller] fn's must have 1 more argument in their ABI than in their MIR",
767 self.get_caller_location(&mut bx
, mir
::SourceInfo { span: fn_span, ..source_info }
);
769 "codegen_call_terminator({:?}): location={:?} (fn_span {:?})",
770 terminator
, location
, fn_span
773 let last_arg
= fn_abi
.args
.last().unwrap();
774 self.codegen_argument(&mut bx
, location
, &mut llargs
, last_arg
);
777 let fn_ptr
= match (llfn
, instance
) {
778 (Some(llfn
), _
) => llfn
,
779 (None
, Some(instance
)) => bx
.get_fn_addr(instance
),
780 _
=> span_bug
!(span
, "no llfn for call"),
789 destination
.as_ref().map(|&(_
, target
)| (ret_dest
, target
)),
794 fn codegen_asm_terminator(
796 helper
: TerminatorCodegenHelper
<'tcx
>,
798 terminator
: &mir
::Terminator
<'tcx
>,
799 template
: &[ast
::InlineAsmTemplatePiece
],
800 operands
: &[mir
::InlineAsmOperand
<'tcx
>],
801 options
: ast
::InlineAsmOptions
,
803 destination
: Option
<mir
::BasicBlock
>,
805 let span
= terminator
.source_info
.span
;
807 let operands
: Vec
<_
> = operands
809 .map(|op
| match *op
{
810 mir
::InlineAsmOperand
::In { reg, ref value }
=> {
811 let value
= self.codegen_operand(&mut bx
, value
);
812 InlineAsmOperandRef
::In { reg, value }
814 mir
::InlineAsmOperand
::Out { reg, late, ref place }
=> {
815 let place
= place
.map(|place
| self.codegen_place(&mut bx
, place
.as_ref()));
816 InlineAsmOperandRef
::Out { reg, late, place }
818 mir
::InlineAsmOperand
::InOut { reg, late, ref in_value, ref out_place }
=> {
819 let in_value
= self.codegen_operand(&mut bx
, in_value
);
821 out_place
.map(|out_place
| self.codegen_place(&mut bx
, out_place
.as_ref()));
822 InlineAsmOperandRef
::InOut { reg, late, in_value, out_place }
824 mir
::InlineAsmOperand
::Const { ref value }
=> {
825 let const_value
= self
826 .eval_mir_constant(value
)
827 .unwrap_or_else(|_
| span_bug
!(span
, "asm const cannot be resolved"));
829 let size
= bx
.layout_of(ty
).size
;
830 let scalar
= match const_value
{
831 ConstValue
::Scalar(s
) => s
,
834 "expected Scalar for promoted asm const, but got {:#?}",
838 let value
= scalar
.assert_bits(size
);
839 let string
= match ty
.kind() {
840 ty
::Uint(_
) => value
.to_string(),
842 match int_ty
.normalize(bx
.tcx().sess
.target
.pointer_width
) {
843 ty
::IntTy
::I8
=> (value
as i8).to_string(),
844 ty
::IntTy
::I16
=> (value
as i16).to_string(),
845 ty
::IntTy
::I32
=> (value
as i32).to_string(),
846 ty
::IntTy
::I64
=> (value
as i64).to_string(),
847 ty
::IntTy
::I128
=> (value
as i128
).to_string(),
848 ty
::IntTy
::Isize
=> unreachable
!(),
851 ty
::Float(ty
::FloatTy
::F32
) => f32::from_bits(value
as u32).to_string(),
852 ty
::Float(ty
::FloatTy
::F64
) => f64::from_bits(value
as u64).to_string(),
853 _
=> span_bug
!(span
, "asm const has bad type {}", ty
),
855 InlineAsmOperandRef
::Const { string }
857 mir
::InlineAsmOperand
::SymFn { ref value }
=> {
858 let literal
= self.monomorphize(value
.literal
);
859 if let ty
::FnDef(def_id
, substs
) = *literal
.ty().kind() {
860 let instance
= ty
::Instance
::resolve_for_fn_ptr(
862 ty
::ParamEnv
::reveal_all(),
867 InlineAsmOperandRef
::SymFn { instance }
869 span_bug
!(span
, "invalid type for asm sym (fn)");
872 mir
::InlineAsmOperand
::SymStatic { def_id }
=> {
873 InlineAsmOperandRef
::SymStatic { def_id }
878 bx
.codegen_inline_asm(template
, &operands
, options
, line_spans
);
880 if let Some(target
) = destination
{
881 helper
.funclet_br(self, &mut bx
, target
);
888 impl<'a
, 'tcx
, Bx
: BuilderMethods
<'a
, 'tcx
>> FunctionCx
<'a
, 'tcx
, Bx
> {
889 pub fn codegen_block(&mut self, bb
: mir
::BasicBlock
) {
890 let mut bx
= self.build_block(bb
);
894 debug
!("codegen_block({:?}={:?})", bb
, data
);
896 for statement
in &data
.statements
{
897 bx
= self.codegen_statement(bx
, statement
);
900 self.codegen_terminator(bx
, bb
, data
.terminator());
903 fn codegen_terminator(
907 terminator
: &'tcx mir
::Terminator
<'tcx
>,
909 debug
!("codegen_terminator: {:?}", terminator
);
911 // Create the cleanup bundle, if needed.
912 let funclet_bb
= self.cleanup_kinds
[bb
].funclet_bb(bb
);
913 let helper
= TerminatorCodegenHelper { bb, terminator, funclet_bb }
;
915 self.set_debug_loc(&mut bx
, terminator
.source_info
);
916 match terminator
.kind
{
917 mir
::TerminatorKind
::Resume
=> self.codegen_resume_terminator(helper
, bx
),
919 mir
::TerminatorKind
::Abort
=> {
921 // `abort` does not terminate the block, so we still need to generate
922 // an `unreachable` terminator after it.
926 mir
::TerminatorKind
::Goto { target }
=> {
928 // This is an unconditional branch back to this same basic block. That means we
929 // have something like a `loop {}` statement. LLVM versions before 12.0
930 // miscompile this because they assume forward progress. For older versions
931 // try to handle just this specific case which comes up commonly in practice
932 // (e.g., in embedded code).
934 // NB: the `sideeffect` currently checks for the LLVM version used internally.
938 helper
.funclet_br(self, &mut bx
, target
);
941 mir
::TerminatorKind
::SwitchInt { ref discr, switch_ty, ref targets }
=> {
942 self.codegen_switchint_terminator(helper
, bx
, discr
, switch_ty
, targets
);
945 mir
::TerminatorKind
::Return
=> {
946 self.codegen_return_terminator(bx
);
949 mir
::TerminatorKind
::Unreachable
=> {
953 mir
::TerminatorKind
::Drop { place, target, unwind }
=> {
954 self.codegen_drop_terminator(helper
, bx
, place
, target
, unwind
);
957 mir
::TerminatorKind
::Assert { ref cond, expected, ref msg, target, cleanup }
=> {
958 self.codegen_assert_terminator(
959 helper
, bx
, terminator
, cond
, expected
, msg
, target
, cleanup
,
963 mir
::TerminatorKind
::DropAndReplace { .. }
=> {
964 bug
!("undesugared DropAndReplace in codegen: {:?}", terminator
);
967 mir
::TerminatorKind
::Call
{
975 self.codegen_call_terminator(
986 mir
::TerminatorKind
::GeneratorDrop
| mir
::TerminatorKind
::Yield { .. }
=> {
987 bug
!("generator ops in codegen")
989 mir
::TerminatorKind
::FalseEdge { .. }
| mir
::TerminatorKind
::FalseUnwind { .. }
=> {
990 bug
!("borrowck false edges in codegen")
993 mir
::TerminatorKind
::InlineAsm
{
1000 self.codegen_asm_terminator(
1014 fn codegen_argument(
1017 op
: OperandRef
<'tcx
, Bx
::Value
>,
1018 llargs
: &mut Vec
<Bx
::Value
>,
1019 arg
: &ArgAbi
<'tcx
, Ty
<'tcx
>>,
1021 // Fill padding with undef value, where applicable.
1022 if let Some(ty
) = arg
.pad
{
1023 llargs
.push(bx
.const_undef(bx
.reg_backend_type(&ty
)))
1026 if arg
.is_ignore() {
1030 if let PassMode
::Pair(..) = arg
.mode
{
1037 _
=> bug
!("codegen_argument: {:?} invalid for pair argument", op
),
1039 } else if arg
.is_unsized_indirect() {
1041 Ref(a
, Some(b
), _
) => {
1046 _
=> bug
!("codegen_argument: {:?} invalid for unsized indirect argument", op
),
1050 // Force by-ref if we have to load through a cast pointer.
1051 let (mut llval
, align
, by_ref
) = match op
.val
{
1052 Immediate(_
) | Pair(..) => match arg
.mode
{
1053 PassMode
::Indirect { .. }
| PassMode
::Cast(_
) => {
1054 let scratch
= PlaceRef
::alloca(bx
, arg
.layout
);
1055 op
.val
.store(bx
, scratch
);
1056 (scratch
.llval
, scratch
.align
, true)
1058 _
=> (op
.immediate_or_packed_pair(bx
), arg
.layout
.align
.abi
, false),
1060 Ref(llval
, _
, align
) => {
1061 if arg
.is_indirect() && align
< arg
.layout
.align
.abi
{
1062 // `foo(packed.large_field)`. We can't pass the (unaligned) field directly. I
1063 // think that ATM (Rust 1.16) we only pass temporaries, but we shouldn't
1064 // have scary latent bugs around.
1066 let scratch
= PlaceRef
::alloca(bx
, arg
.layout
);
1076 (scratch
.llval
, scratch
.align
, true)
1078 (llval
, align
, true)
1083 if by_ref
&& !arg
.is_indirect() {
1084 // Have to load the argument, maybe while casting it.
1085 if let PassMode
::Cast(ty
) = arg
.mode
{
1086 let addr
= bx
.pointercast(llval
, bx
.type_ptr_to(bx
.cast_backend_type(&ty
)));
1087 llval
= bx
.load(addr
, align
.min(arg
.layout
.align
.abi
));
1089 // We can't use `PlaceRef::load` here because the argument
1090 // may have a type we don't treat as immediate, but the ABI
1091 // used for this call is passing it by-value. In that case,
1092 // the load would just produce `OperandValue::Ref` instead
1093 // of the `OperandValue::Immediate` we need for the call.
1094 llval
= bx
.load(llval
, align
);
1095 if let abi
::Abi
::Scalar(ref scalar
) = arg
.layout
.abi
{
1096 if scalar
.is_bool() {
1097 bx
.range_metadata(llval
, 0..2);
1100 // We store bools as `i8` so we need to truncate to `i1`.
1101 llval
= bx
.to_immediate(llval
, arg
.layout
);
1108 fn codegen_arguments_untupled(
1111 operand
: &mir
::Operand
<'tcx
>,
1112 llargs
: &mut Vec
<Bx
::Value
>,
1113 args
: &[ArgAbi
<'tcx
, Ty
<'tcx
>>],
1115 let tuple
= self.codegen_operand(bx
, operand
);
1117 // Handle both by-ref and immediate tuples.
1118 if let Ref(llval
, None
, align
) = tuple
.val
{
1119 let tuple_ptr
= PlaceRef
::new_sized_aligned(llval
, tuple
.layout
, align
);
1120 for i
in 0..tuple
.layout
.fields
.count() {
1121 let field_ptr
= tuple_ptr
.project_field(bx
, i
);
1122 let field
= bx
.load_operand(field_ptr
);
1123 self.codegen_argument(bx
, field
, llargs
, &args
[i
]);
1125 } else if let Ref(_
, Some(_
), _
) = tuple
.val
{
1126 bug
!("closure arguments must be sized")
1128 // If the tuple is immediate, the elements are as well.
1129 for i
in 0..tuple
.layout
.fields
.count() {
1130 let op
= tuple
.extract_field(bx
, i
);
1131 self.codegen_argument(bx
, op
, llargs
, &args
[i
]);
1136 fn get_caller_location(
1139 mut source_info
: mir
::SourceInfo
,
1140 ) -> OperandRef
<'tcx
, Bx
::Value
> {
1143 let mut span_to_caller_location
= |span
: Span
| {
1144 let topmost
= span
.ctxt().outer_expn().expansion_cause().unwrap_or(span
);
1145 let caller
= tcx
.sess
.source_map().lookup_char_pos(topmost
.lo());
1146 let const_loc
= tcx
.const_caller_location((
1147 Symbol
::intern(&caller
.file
.name
.to_string()),
1149 caller
.col_display
as u32 + 1,
1151 OperandRef
::from_const(bx
, const_loc
, bx
.tcx().caller_location_ty())
1154 // Walk up the `SourceScope`s, in case some of them are from MIR inlining.
1155 // If so, the starting `source_info.span` is in the innermost inlined
1156 // function, and will be replaced with outer callsite spans as long
1157 // as the inlined functions were `#[track_caller]`.
1159 let scope_data
= &self.mir
.source_scopes
[source_info
.scope
];
1161 if let Some((callee
, callsite_span
)) = scope_data
.inlined
{
1162 // Stop inside the most nested non-`#[track_caller]` function,
1163 // before ever reaching its caller (which is irrelevant).
1164 if !callee
.def
.requires_caller_location(tcx
) {
1165 return span_to_caller_location(source_info
.span
);
1167 source_info
.span
= callsite_span
;
1170 // Skip past all of the parents with `inlined: None`.
1171 match scope_data
.inlined_parent_scope
{
1172 Some(parent
) => source_info
.scope
= parent
,
1177 // No inlined `SourceScope`s, or all of them were `#[track_caller]`.
1178 self.caller_location
.unwrap_or_else(|| span_to_caller_location(source_info
.span
))
1181 fn get_personality_slot(&mut self, bx
: &mut Bx
) -> PlaceRef
<'tcx
, Bx
::Value
> {
1183 if let Some(slot
) = self.personality_slot
{
1186 let layout
= cx
.layout_of(
1187 cx
.tcx().intern_tup(&[cx
.tcx().mk_mut_ptr(cx
.tcx().types
.u8), cx
.tcx().types
.i32]),
1189 let slot
= PlaceRef
::alloca(bx
, layout
);
1190 self.personality_slot
= Some(slot
);
1195 /// Returns the landing-pad wrapper around the given basic block.
1197 /// No-op in MSVC SEH scheme.
1198 fn landing_pad_to(&mut self, target_bb
: mir
::BasicBlock
) -> Bx
::BasicBlock
{
1199 if let Some(block
) = self.landing_pads
[target_bb
] {
1203 let block
= self.blocks
[target_bb
];
1204 let landing_pad
= self.landing_pad_uncached(block
);
1205 self.landing_pads
[target_bb
] = Some(landing_pad
);
1209 fn landing_pad_uncached(&mut self, target_bb
: Bx
::BasicBlock
) -> Bx
::BasicBlock
{
1210 if base
::wants_msvc_seh(self.cx
.sess()) {
1211 span_bug
!(self.mir
.span
, "landing pad was not inserted?")
1214 let mut bx
= self.new_block("cleanup");
1216 let llpersonality
= self.cx
.eh_personality();
1217 let llretty
= self.landing_pad_type();
1218 let lp
= bx
.landing_pad(llretty
, llpersonality
, 1);
1221 let slot
= self.get_personality_slot(&mut bx
);
1222 slot
.storage_live(&mut bx
);
1223 Pair(bx
.extract_value(lp
, 0), bx
.extract_value(lp
, 1)).store(&mut bx
, slot
);
1229 fn landing_pad_type(&self) -> Bx
::Type
{
1231 cx
.type_struct(&[cx
.type_i8p(), cx
.type_i32()], false)
1234 fn unreachable_block(&mut self) -> Bx
::BasicBlock
{
1235 self.unreachable_block
.unwrap_or_else(|| {
1236 let mut bx
= self.new_block("unreachable");
1238 self.unreachable_block
= Some(bx
.llbb());
1243 pub fn new_block(&self, name
: &str) -> Bx
{
1244 Bx
::new_block(self.cx
, self.llfn
, name
)
1247 pub fn build_block(&self, bb
: mir
::BasicBlock
) -> Bx
{
1248 let mut bx
= Bx
::with_cx(self.cx
);
1249 bx
.position_at_end(self.blocks
[bb
]);
1253 fn make_return_dest(
1256 dest
: mir
::Place
<'tcx
>,
1257 fn_ret
: &ArgAbi
<'tcx
, Ty
<'tcx
>>,
1258 llargs
: &mut Vec
<Bx
::Value
>,
1260 ) -> ReturnDest
<'tcx
, Bx
::Value
> {
1261 // If the return is ignored, we can just return a do-nothing `ReturnDest`.
1262 if fn_ret
.is_ignore() {
1263 return ReturnDest
::Nothing
;
1265 let dest
= if let Some(index
) = dest
.as_local() {
1266 match self.locals
[index
] {
1267 LocalRef
::Place(dest
) => dest
,
1268 LocalRef
::UnsizedPlace(_
) => bug
!("return type must be sized"),
1269 LocalRef
::Operand(None
) => {
1270 // Handle temporary places, specifically `Operand` ones, as
1271 // they don't have `alloca`s.
1272 return if fn_ret
.is_indirect() {
1273 // Odd, but possible, case, we have an operand temporary,
1274 // but the calling convention has an indirect return.
1275 let tmp
= PlaceRef
::alloca(bx
, fn_ret
.layout
);
1276 tmp
.storage_live(bx
);
1277 llargs
.push(tmp
.llval
);
1278 ReturnDest
::IndirectOperand(tmp
, index
)
1279 } else if is_intrinsic
{
1280 // Currently, intrinsics always need a location to store
1281 // the result, so we create a temporary `alloca` for the
1283 let tmp
= PlaceRef
::alloca(bx
, fn_ret
.layout
);
1284 tmp
.storage_live(bx
);
1285 ReturnDest
::IndirectOperand(tmp
, index
)
1287 ReturnDest
::DirectOperand(index
)
1290 LocalRef
::Operand(Some(_
)) => {
1291 bug
!("place local already assigned to");
1297 mir
::PlaceRef { local: dest.local, projection: &dest.projection }
,
1300 if fn_ret
.is_indirect() {
1301 if dest
.align
< dest
.layout
.align
.abi
{
1302 // Currently, MIR code generation does not create calls
1303 // that store directly to fields of packed structs (in
1304 // fact, the calls it creates write only to temps).
1306 // If someone changes that, please update this code path
1307 // to create a temporary.
1308 span_bug
!(self.mir
.span
, "can't directly store to unaligned value");
1310 llargs
.push(dest
.llval
);
1313 ReturnDest
::Store(dest
)
1317 fn codegen_transmute(&mut self, bx
: &mut Bx
, src
: &mir
::Operand
<'tcx
>, dst
: mir
::Place
<'tcx
>) {
1318 if let Some(index
) = dst
.as_local() {
1319 match self.locals
[index
] {
1320 LocalRef
::Place(place
) => self.codegen_transmute_into(bx
, src
, place
),
1321 LocalRef
::UnsizedPlace(_
) => bug
!("transmute must not involve unsized locals"),
1322 LocalRef
::Operand(None
) => {
1323 let dst_layout
= bx
.layout_of(self.monomorphized_place_ty(dst
.as_ref()));
1324 assert
!(!dst_layout
.ty
.has_erasable_regions());
1325 let place
= PlaceRef
::alloca(bx
, dst_layout
);
1326 place
.storage_live(bx
);
1327 self.codegen_transmute_into(bx
, src
, place
);
1328 let op
= bx
.load_operand(place
);
1329 place
.storage_dead(bx
);
1330 self.locals
[index
] = LocalRef
::Operand(Some(op
));
1331 self.debug_introduce_local(bx
, index
);
1333 LocalRef
::Operand(Some(op
)) => {
1334 assert
!(op
.layout
.is_zst(), "assigning to initialized SSAtemp");
1338 let dst
= self.codegen_place(bx
, dst
.as_ref());
1339 self.codegen_transmute_into(bx
, src
, dst
);
1343 fn codegen_transmute_into(
1346 src
: &mir
::Operand
<'tcx
>,
1347 dst
: PlaceRef
<'tcx
, Bx
::Value
>,
1349 let src
= self.codegen_operand(bx
, src
);
1351 // Special-case transmutes between scalars as simple bitcasts.
1352 match (&src
.layout
.abi
, &dst
.layout
.abi
) {
1353 (abi
::Abi
::Scalar(src_scalar
), abi
::Abi
::Scalar(dst_scalar
)) => {
1354 // HACK(eddyb) LLVM doesn't like `bitcast`s between pointers and non-pointers.
1355 if (src_scalar
.value
== abi
::Pointer
) == (dst_scalar
.value
== abi
::Pointer
) {
1356 assert_eq
!(src
.layout
.size
, dst
.layout
.size
);
1358 // NOTE(eddyb) the `from_immediate` and `to_immediate_scalar`
1359 // conversions allow handling `bool`s the same as `u8`s.
1360 let src
= bx
.from_immediate(src
.immediate());
1361 let src_as_dst
= bx
.bitcast(src
, bx
.backend_type(dst
.layout
));
1362 Immediate(bx
.to_immediate_scalar(src_as_dst
, dst_scalar
)).store(bx
, dst
);
1369 let llty
= bx
.backend_type(src
.layout
);
1370 let cast_ptr
= bx
.pointercast(dst
.llval
, bx
.type_ptr_to(llty
));
1371 let align
= src
.layout
.align
.abi
.min(dst
.align
);
1372 src
.val
.store(bx
, PlaceRef
::new_sized_aligned(cast_ptr
, src
.layout
, align
));
1375 // Stores the return value of a function call into it's final location.
1379 dest
: ReturnDest
<'tcx
, Bx
::Value
>,
1380 ret_abi
: &ArgAbi
<'tcx
, Ty
<'tcx
>>,
1383 use self::ReturnDest
::*;
1387 Store(dst
) => bx
.store_arg(&ret_abi
, llval
, dst
),
1388 IndirectOperand(tmp
, index
) => {
1389 let op
= bx
.load_operand(tmp
);
1390 tmp
.storage_dead(bx
);
1391 self.locals
[index
] = LocalRef
::Operand(Some(op
));
1392 self.debug_introduce_local(bx
, index
);
1394 DirectOperand(index
) => {
1395 // If there is a cast, we have to store and reload.
1396 let op
= if let PassMode
::Cast(_
) = ret_abi
.mode
{
1397 let tmp
= PlaceRef
::alloca(bx
, ret_abi
.layout
);
1398 tmp
.storage_live(bx
);
1399 bx
.store_arg(&ret_abi
, llval
, tmp
);
1400 let op
= bx
.load_operand(tmp
);
1401 tmp
.storage_dead(bx
);
1404 OperandRef
::from_immediate_or_packed_pair(bx
, llval
, ret_abi
.layout
)
1406 self.locals
[index
] = LocalRef
::Operand(Some(op
));
1407 self.debug_introduce_local(bx
, index
);
1413 enum ReturnDest
<'tcx
, V
> {
1414 // Do nothing; the return value is indirect or ignored.
1416 // Store the return value to the pointer.
1417 Store(PlaceRef
<'tcx
, V
>),
1418 // Store an indirect return value to an operand local place.
1419 IndirectOperand(PlaceRef
<'tcx
, V
>, mir
::Local
),
1420 // Store a direct return value to an operand local place.
1421 DirectOperand(mir
::Local
),