1 //! Codegen of a single function
3 use rustc_index
::vec
::IndexVec
;
4 use rustc_middle
::ty
::adjustment
::PointerCast
;
8 pub(crate) fn codegen_fn
<'tcx
>(
9 cx
: &mut crate::CodegenCx
<'tcx
, impl Module
>,
10 instance
: Instance
<'tcx
>,
16 crate::PrintOnPanic(|| format
!("{:?} {}", instance
, tcx
.symbol_name(instance
).name
));
17 debug_assert
!(!instance
.substs
.needs_infer());
19 let mir
= tcx
.instance_mir(instance
.def
);
22 let (name
, sig
) = get_function_name_and_sig(tcx
, cx
.module
.isa().triple(), instance
, false);
23 let func_id
= cx
.module
.declare_function(&name
, linkage
, &sig
).unwrap();
25 cx
.cached_context
.clear();
27 // Make the FunctionBuilder
28 let mut func_ctx
= FunctionBuilderContext
::new();
29 let mut func
= std
::mem
::replace(&mut cx
.cached_context
.func
, Function
::new());
30 func
.name
= ExternalName
::user(0, func_id
.as_u32());
32 func
.collect_debug_info();
34 let mut bcx
= FunctionBuilder
::new(&mut func
, &mut func_ctx
);
37 let start_block
= bcx
.create_block();
38 let block_map
: IndexVec
<BasicBlock
, Block
> = (0..mir
.basic_blocks().len())
39 .map(|_
| bcx
.create_block())
43 let pointer_type
= cx
.module
.target_config().pointer_type();
44 let clif_comments
= crate::pretty_clif
::CommentWriter
::new(tcx
, instance
);
46 let mut fx
= FunctionCx
{
56 local_map
: IndexVec
::with_capacity(mir
.local_decls
.len()),
57 caller_location
: None
, // set by `codegen_fn_prelude`
58 cold_blocks
: EntitySet
::new(),
61 source_info_set
: indexmap
::IndexSet
::new(),
67 let arg_uninhabited
= fx
.mir
.args_iter().any(|arg
| {
68 fx
.layout_of(fx
.monomorphize(&fx
.mir
.local_decls
[arg
].ty
))
75 .append_block_params_for_function_params(fx
.block_map
[START_BLOCK
]);
76 fx
.bcx
.switch_to_block(fx
.block_map
[START_BLOCK
]);
77 crate::trap
::trap_unreachable(&mut fx
, "function has uninhabited argument");
79 tcx
.sess
.time("codegen clif ir", || {
80 tcx
.sess
.time("codegen prelude", || {
81 crate::abi
::codegen_fn_prelude(&mut fx
, start_block
)
83 codegen_fn_content(&mut fx
);
87 // Recover all necessary data from fx, before accessing func will prevent future access to it.
88 let instance
= fx
.instance
;
89 let mut clif_comments
= fx
.clif_comments
;
90 let source_info_set
= fx
.source_info_set
;
91 let local_map
= fx
.local_map
;
92 let cold_blocks
= fx
.cold_blocks
;
94 // Store function in context
95 let context
= &mut cx
.cached_context
;
98 crate::pretty_clif
::write_clif_file(tcx
, "unopt", None
, instance
, &context
, &clif_comments
);
101 verify_func(tcx
, &clif_comments
, &context
.func
);
103 // Perform rust specific optimizations
104 tcx
.sess
.time("optimize clif ir", || {
105 crate::optimize
::optimize_function(
114 // If the return block is not reachable, then the SSA builder may have inserted an `iconst.i128`
115 // instruction, which doesn't have an encoding.
116 context
.compute_cfg();
117 context
.compute_domtree();
118 context
.eliminate_unreachable_code(cx
.module
.isa()).unwrap();
119 context
.dce(cx
.module
.isa()).unwrap();
122 let module
= &mut cx
.module
;
123 tcx
.sess
.time("define function", || {
128 &mut cranelift_codegen
::binemit
::NullTrapSink {}
,
133 // Write optimized function to file for debugging
134 crate::pretty_clif
::write_clif_file(
137 Some(cx
.module
.isa()),
143 // Define debuginfo for function
144 let isa
= cx
.module
.isa();
145 let debug_context
= &mut cx
.debug_context
;
146 let unwind_context
= &mut cx
.unwind_context
;
147 tcx
.sess
.time("generate debug info", || {
148 if let Some(debug_context
) = debug_context
{
149 debug_context
.define_function(
159 unwind_context
.add_function(func_id
, &context
, isa
);
162 // Clear context to make it usable for the next function
166 pub(crate) fn verify_func(
168 writer
: &crate::pretty_clif
::CommentWriter
,
171 tcx
.sess
.time("verify clif ir", || {
172 let flags
= cranelift_codegen
::settings
::Flags
::new(cranelift_codegen
::settings
::builder());
173 match cranelift_codegen
::verify_function(&func
, &flags
) {
176 tcx
.sess
.err(&format
!("{:?}", err
));
177 let pretty_error
= cranelift_codegen
::print_errors
::pretty_verifier_error(
180 Some(Box
::new(writer
)),
184 .fatal(&format
!("cranelift verify error:\n{}", pretty_error
));
190 fn codegen_fn_content(fx
: &mut FunctionCx
<'_
, '_
, impl Module
>) {
191 crate::constant
::check_constants(fx
);
193 for (bb
, bb_data
) in fx
.mir
.basic_blocks().iter_enumerated() {
194 let block
= fx
.get_block(bb
);
195 fx
.bcx
.switch_to_block(block
);
197 if bb_data
.is_cleanup
{
198 // Unwinding after panicking is not supported
201 // FIXME once unwinding is supported uncomment next lines
202 // // Unwinding is unlikely to happen, so mark cleanup block's as cold.
203 // fx.cold_blocks.insert(block);
207 for stmt
in &bb_data
.statements
{
208 fx
.set_debug_loc(stmt
.source_info
);
209 codegen_stmt(fx
, block
, stmt
);
212 #[cfg(debug_assertions)]
214 let mut terminator_head
= "\n".to_string();
218 .fmt_head(&mut terminator_head
)
220 let inst
= fx
.bcx
.func
.layout
.last_inst(block
).unwrap();
221 fx
.add_comment(inst
, terminator_head
);
224 fx
.set_debug_loc(bb_data
.terminator().source_info
);
226 match &bb_data
.terminator().kind
{
227 TerminatorKind
::Goto { target }
=> {
228 if let TerminatorKind
::Return
= fx
.mir
[*target
].terminator().kind
{
229 let mut can_immediately_return
= true;
230 for stmt
in &fx
.mir
[*target
].statements
{
231 if let StatementKind
::StorageDead(_
) = stmt
.kind
{
233 // FIXME Can sometimes happen, see rust-lang/rust#70531
234 can_immediately_return
= false;
239 if can_immediately_return
{
240 crate::abi
::codegen_return(fx
);
245 let block
= fx
.get_block(*target
);
246 fx
.bcx
.ins().jump(block
, &[]);
248 TerminatorKind
::Return
=> {
249 crate::abi
::codegen_return(fx
);
251 TerminatorKind
::Assert
{
258 if !fx
.tcx
.sess
.overflow_checks() {
259 if let mir
::AssertKind
::OverflowNeg(_
) = *msg
{
260 let target
= fx
.get_block(*target
);
261 fx
.bcx
.ins().jump(target
, &[]);
265 let cond
= codegen_operand(fx
, cond
).load_scalar(fx
);
267 let target
= fx
.get_block(*target
);
268 let failure
= fx
.bcx
.create_block();
269 fx
.cold_blocks
.insert(failure
);
272 fx
.bcx
.ins().brz(cond
, failure
, &[]);
274 fx
.bcx
.ins().brnz(cond
, failure
, &[]);
276 fx
.bcx
.ins().jump(target
, &[]);
278 fx
.bcx
.switch_to_block(failure
);
282 AssertKind
::BoundsCheck { ref len, ref index }
=> {
283 let len
= codegen_operand(fx
, len
).load_scalar(fx
);
284 let index
= codegen_operand(fx
, index
).load_scalar(fx
);
286 .get_caller_location(bb_data
.terminator().source_info
.span
)
291 rustc_hir
::LangItem
::PanicBoundsCheck
,
292 &[index
, len
, location
],
293 bb_data
.terminator().source_info
.span
,
297 let msg_str
= msg
.description();
298 codegen_panic(fx
, msg_str
, bb_data
.terminator().source_info
.span
);
303 TerminatorKind
::SwitchInt
{
308 let discr
= codegen_operand(fx
, discr
).load_scalar(fx
);
310 if switch_ty
.kind() == fx
.tcx
.types
.bool
.kind() {
311 assert_eq
!(targets
.iter().count(), 1);
312 let (then_value
, then_block
) = targets
.iter().next().unwrap();
313 let then_block
= fx
.get_block(then_block
);
314 let else_block
= fx
.get_block(targets
.otherwise());
315 let test_zero
= match then_value
{
318 _
=> unreachable
!("{:?}", targets
),
321 let discr
= crate::optimize
::peephole
::maybe_unwrap_bint(&mut fx
.bcx
, discr
);
322 let (discr
, is_inverted
) =
323 crate::optimize
::peephole
::maybe_unwrap_bool_not(&mut fx
.bcx
, discr
);
324 let test_zero
= if is_inverted { !test_zero }
else { test_zero }
;
325 let discr
= crate::optimize
::peephole
::maybe_unwrap_bint(&mut fx
.bcx
, discr
);
327 crate::optimize
::peephole
::make_branchable_value(&mut fx
.bcx
, discr
);
329 fx
.bcx
.ins().brz(discr
, then_block
, &[]);
330 fx
.bcx
.ins().jump(else_block
, &[]);
332 fx
.bcx
.ins().brnz(discr
, then_block
, &[]);
333 fx
.bcx
.ins().jump(else_block
, &[]);
336 let mut switch
= ::cranelift_frontend
::Switch
::new();
337 for (value
, block
) in targets
.iter() {
338 let block
= fx
.get_block(block
);
339 switch
.set_entry(value
, block
);
341 let otherwise_block
= fx
.get_block(targets
.otherwise());
342 switch
.emit(&mut fx
.bcx
, discr
, otherwise_block
);
345 TerminatorKind
::Call
{
353 fx
.tcx
.sess
.time("codegen call", || {
354 crate::abi
::codegen_terminator_call(
364 TerminatorKind
::InlineAsm
{
371 crate::inline_asm
::codegen_inline_asm(
373 bb_data
.terminator().source_info
.span
,
380 Some(destination
) => {
381 let destination_block
= fx
.get_block(destination
);
382 fx
.bcx
.ins().jump(destination_block
, &[]);
385 crate::trap
::trap_unreachable(
387 "[corruption] Returned from noreturn inline asm",
392 TerminatorKind
::Resume
| TerminatorKind
::Abort
=> {
393 trap_unreachable(fx
, "[corruption] Unwinding bb reached.");
395 TerminatorKind
::Unreachable
=> {
396 trap_unreachable(fx
, "[corruption] Hit unreachable code.");
398 TerminatorKind
::Yield { .. }
399 | TerminatorKind
::FalseEdge { .. }
400 | TerminatorKind
::FalseUnwind { .. }
401 | TerminatorKind
::DropAndReplace { .. }
402 | TerminatorKind
::GeneratorDrop
=> {
403 bug
!("shouldn't exist at codegen {:?}", bb_data
.terminator());
405 TerminatorKind
::Drop
{
410 let drop_place
= codegen_place(fx
, *place
);
411 crate::abi
::codegen_drop(fx
, bb_data
.terminator().source_info
.span
, drop_place
);
413 let target_block
= fx
.get_block(*target
);
414 fx
.bcx
.ins().jump(target_block
, &[]);
419 fx
.bcx
.seal_all_blocks();
423 fn codegen_stmt
<'tcx
>(
424 fx
: &mut FunctionCx
<'_
, 'tcx
, impl Module
>,
425 #[allow(unused_variables)] cur_block: Block,
426 stmt
: &Statement
<'tcx
>,
428 let _print_guard
= crate::PrintOnPanic(|| format
!("stmt {:?}", stmt
));
430 fx
.set_debug_loc(stmt
.source_info
);
432 #[cfg(false_debug_assertions)]
434 StatementKind
::StorageLive(..) | StatementKind
::StorageDead(..) => {}
// Those are not very useful
436 let inst
= fx
.bcx
.func
.layout
.last_inst(cur_block
).unwrap();
437 fx
.add_comment(inst
, format
!("{:?}", stmt
));
442 StatementKind
::SetDiscriminant
{
446 let place
= codegen_place(fx
, **place
);
447 crate::discriminant
::codegen_set_discriminant(fx
, place
, *variant_index
);
449 StatementKind
::Assign(to_place_and_rval
) => {
450 let lval
= codegen_place(fx
, to_place_and_rval
.0);
451 let dest_layout
= lval
.layout();
452 match to_place_and_rval
.1 {
453 Rvalue
::Use(ref operand
) => {
454 let val
= codegen_operand(fx
, operand
);
455 lval
.write_cvalue(fx
, val
);
457 Rvalue
::Ref(_
, _
, place
) | Rvalue
::AddressOf(_
, place
) => {
458 let place
= codegen_place(fx
, place
);
459 let ref_
= place
.place_ref(fx
, lval
.layout());
460 lval
.write_cvalue(fx
, ref_
);
462 Rvalue
::ThreadLocalRef(def_id
) => {
463 let val
= crate::constant
::codegen_tls_ref(fx
, def_id
, lval
.layout());
464 lval
.write_cvalue(fx
, val
);
466 Rvalue
::BinaryOp(bin_op
, ref lhs
, ref rhs
) => {
467 let lhs
= codegen_operand(fx
, lhs
);
468 let rhs
= codegen_operand(fx
, rhs
);
470 let res
= crate::num
::codegen_binop(fx
, bin_op
, lhs
, rhs
);
471 lval
.write_cvalue(fx
, res
);
473 Rvalue
::CheckedBinaryOp(bin_op
, ref lhs
, ref rhs
) => {
474 let lhs
= codegen_operand(fx
, lhs
);
475 let rhs
= codegen_operand(fx
, rhs
);
477 let res
= if !fx
.tcx
.sess
.overflow_checks() {
479 crate::num
::codegen_int_binop(fx
, bin_op
, lhs
, rhs
).load_scalar(fx
);
480 let is_overflow
= fx
.bcx
.ins().iconst(types
::I8
, 0);
481 CValue
::by_val_pair(val
, is_overflow
, lval
.layout())
483 crate::num
::codegen_checked_int_binop(fx
, bin_op
, lhs
, rhs
)
486 lval
.write_cvalue(fx
, res
);
488 Rvalue
::UnaryOp(un_op
, ref operand
) => {
489 let operand
= codegen_operand(fx
, operand
);
490 let layout
= operand
.layout();
491 let val
= operand
.load_scalar(fx
);
492 let res
= match un_op
{
493 UnOp
::Not
=> match layout
.ty
.kind() {
495 let res
= fx
.bcx
.ins().icmp_imm(IntCC
::Equal
, val
, 0);
496 CValue
::by_val(fx
.bcx
.ins().bint(types
::I8
, res
), layout
)
498 ty
::Uint(_
) | ty
::Int(_
) => {
499 CValue
::by_val(fx
.bcx
.ins().bnot(val
), layout
)
501 _
=> unreachable
!("un op Not for {:?}", layout
.ty
),
503 UnOp
::Neg
=> match layout
.ty
.kind() {
504 ty
::Int(IntTy
::I128
) => {
505 // FIXME remove this case once ineg.i128 works
507 CValue
::const_val(fx
, layout
, ty
::ScalarInt
::null(layout
.size
));
508 crate::num
::codegen_int_binop(fx
, BinOp
::Sub
, zero
, operand
)
510 ty
::Int(_
) => CValue
::by_val(fx
.bcx
.ins().ineg(val
), layout
),
511 ty
::Float(_
) => CValue
::by_val(fx
.bcx
.ins().fneg(val
), layout
),
512 _
=> unreachable
!("un op Neg for {:?}", layout
.ty
),
515 lval
.write_cvalue(fx
, res
);
518 CastKind
::Pointer(PointerCast
::ReifyFnPointer
),
522 let from_ty
= fx
.monomorphize(operand
.ty(&fx
.mir
.local_decls
, fx
.tcx
));
523 let to_layout
= fx
.layout_of(fx
.monomorphize(to_ty
));
524 match *from_ty
.kind() {
525 ty
::FnDef(def_id
, substs
) => {
526 let func_ref
= fx
.get_function_ref(
527 Instance
::resolve_for_fn_ptr(
529 ParamEnv
::reveal_all(),
534 .polymorphize(fx
.tcx
),
536 let func_addr
= fx
.bcx
.ins().func_addr(fx
.pointer_type
, func_ref
);
537 lval
.write_cvalue(fx
, CValue
::by_val(func_addr
, to_layout
));
539 _
=> bug
!("Trying to ReifyFnPointer on non FnDef {:?}", from_ty
),
543 CastKind
::Pointer(PointerCast
::UnsafeFnPointer
),
548 CastKind
::Pointer(PointerCast
::MutToConstPointer
),
553 CastKind
::Pointer(PointerCast
::ArrayToPointer
),
557 let to_layout
= fx
.layout_of(fx
.monomorphize(to_ty
));
558 let operand
= codegen_operand(fx
, operand
);
559 lval
.write_cvalue(fx
, operand
.cast_pointer_to(to_layout
));
561 Rvalue
::Cast(CastKind
::Misc
, ref operand
, to_ty
) => {
562 let operand
= codegen_operand(fx
, operand
);
563 let from_ty
= operand
.layout().ty
;
564 let to_ty
= fx
.monomorphize(to_ty
);
567 fx
: &FunctionCx
<'_
, 'tcx
, impl Module
>,
570 ty
.builtin_deref(true)
576 has_ptr_meta(fx
.tcx
, pointee_ty
)
582 if is_fat_ptr(fx
, from_ty
) {
583 if is_fat_ptr(fx
, to_ty
) {
584 // fat-ptr -> fat-ptr
585 lval
.write_cvalue(fx
, operand
.cast_pointer_to(dest_layout
));
587 // fat-ptr -> thin-ptr
588 let (ptr
, _extra
) = operand
.load_scalar_pair(fx
);
589 lval
.write_cvalue(fx
, CValue
::by_val(ptr
, dest_layout
))
591 } else if let ty
::Adt(adt_def
, _substs
) = from_ty
.kind() {
592 // enum -> discriminant value
593 assert
!(adt_def
.is_enum());
595 ty
::Uint(_
) | ty
::Int(_
) => {}
596 _
=> unreachable
!("cast adt {} -> {}", from_ty
, to_ty
),
599 use rustc_target
::abi
::{Int, TagEncoding, Variants}
;
601 match operand
.layout().variants
{
602 Variants
::Single { index }
=> {
606 .discriminant_for_variant(fx
.tcx
, index
)
608 let discr
= if discr
.ty
.is_signed() {
609 fx
.layout_of(discr
.ty
).size
.sign_extend(discr
.val
)
613 let discr
= discr
.into();
615 let discr
= CValue
::const_val(fx
, fx
.layout_of(to_ty
), discr
);
616 lval
.write_cvalue(fx
, discr
);
621 tag_encoding
: TagEncoding
::Direct
,
624 let cast_to
= fx
.clif_type(dest_layout
.ty
).unwrap();
626 // Read the tag/niche-encoded discriminant from memory.
628 operand
.value_field(fx
, mir
::Field
::new(tag_field
));
629 let encoded_discr
= encoded_discr
.load_scalar(fx
);
631 // Decode the discriminant (specifically if it's niche-encoded).
632 let signed
= match tag
.value
{
633 Int(_
, signed
) => signed
,
636 let val
= clif_intcast(fx
, encoded_discr
, cast_to
, signed
);
637 let val
= CValue
::by_val(val
, dest_layout
);
638 lval
.write_cvalue(fx
, val
);
640 Variants
::Multiple { .. }
=> unreachable
!(),
643 let to_clif_ty
= fx
.clif_type(to_ty
).unwrap();
644 let from
= operand
.load_scalar(fx
);
646 let res
= clif_int_or_float_cast(
653 lval
.write_cvalue(fx
, CValue
::by_val(res
, dest_layout
));
657 CastKind
::Pointer(PointerCast
::ClosureFnPointer(_
)),
661 let operand
= codegen_operand(fx
, operand
);
662 match *operand
.layout().ty
.kind() {
663 ty
::Closure(def_id
, substs
) => {
664 let instance
= Instance
::resolve_closure(
668 ty
::ClosureKind
::FnOnce
,
670 .polymorphize(fx
.tcx
);
671 let func_ref
= fx
.get_function_ref(instance
);
672 let func_addr
= fx
.bcx
.ins().func_addr(fx
.pointer_type
, func_ref
);
673 lval
.write_cvalue(fx
, CValue
::by_val(func_addr
, lval
.layout()));
675 _
=> bug
!("{} cannot be cast to a fn ptr", operand
.layout().ty
),
678 Rvalue
::Cast(CastKind
::Pointer(PointerCast
::Unsize
), ref operand
, _to_ty
) => {
679 let operand
= codegen_operand(fx
, operand
);
680 operand
.unsize_value(fx
, lval
);
682 Rvalue
::Discriminant(place
) => {
683 let place
= codegen_place(fx
, place
);
684 let value
= place
.to_cvalue(fx
);
686 crate::discriminant
::codegen_get_discriminant(fx
, value
, dest_layout
);
687 lval
.write_cvalue(fx
, discr
);
689 Rvalue
::Repeat(ref operand
, times
) => {
690 let operand
= codegen_operand(fx
, operand
);
693 .eval(fx
.tcx
, ParamEnv
::reveal_all())
695 .try_to_bits(fx
.tcx
.data_layout
.pointer_size
)
697 if fx
.clif_type(operand
.layout().ty
) == Some(types
::I8
) {
698 let times
= fx
.bcx
.ins().iconst(fx
.pointer_type
, times
as i64);
699 // FIXME use emit_small_memset where possible
700 let addr
= lval
.to_ptr().get_addr(fx
);
701 let val
= operand
.load_scalar(fx
);
703 .call_memset(fx
.cx
.module
.target_config(), addr
, val
, times
);
705 let loop_block
= fx
.bcx
.create_block();
706 let loop_block2
= fx
.bcx
.create_block();
707 let done_block
= fx
.bcx
.create_block();
708 let index
= fx
.bcx
.append_block_param(loop_block
, fx
.pointer_type
);
709 let zero
= fx
.bcx
.ins().iconst(fx
.pointer_type
, 0);
710 fx
.bcx
.ins().jump(loop_block
, &[zero
]);
712 fx
.bcx
.switch_to_block(loop_block
);
713 let done
= fx
.bcx
.ins().icmp_imm(IntCC
::Equal
, index
, times
as i64);
714 fx
.bcx
.ins().brnz(done
, done_block
, &[]);
715 fx
.bcx
.ins().jump(loop_block2
, &[]);
717 fx
.bcx
.switch_to_block(loop_block2
);
718 let to
= lval
.place_index(fx
, index
);
719 to
.write_cvalue(fx
, operand
);
720 let index
= fx
.bcx
.ins().iadd_imm(index
, 1);
721 fx
.bcx
.ins().jump(loop_block
, &[index
]);
723 fx
.bcx
.switch_to_block(done_block
);
727 Rvalue
::Len(place
) => {
728 let place
= codegen_place(fx
, place
);
729 let usize_layout
= fx
.layout_of(fx
.tcx
.types
.usize);
730 let len
= codegen_array_len(fx
, place
);
731 lval
.write_cvalue(fx
, CValue
::by_val(len
, usize_layout
));
733 Rvalue
::NullaryOp(NullOp
::Box
, content_ty
) => {
734 let usize_type
= fx
.clif_type(fx
.tcx
.types
.usize).unwrap();
735 let content_ty
= fx
.monomorphize(content_ty
);
736 let layout
= fx
.layout_of(content_ty
);
737 let llsize
= fx
.bcx
.ins().iconst(usize_type
, layout
.size
.bytes() as i64);
741 .iconst(usize_type
, layout
.align
.abi
.bytes() as i64);
742 let box_layout
= fx
.layout_of(fx
.tcx
.mk_box(content_ty
));
745 let def_id
= match fx
748 .require(rustc_hir
::LangItem
::ExchangeMalloc
)
754 .fatal(&format
!("allocation of `{}` {}", box_layout
.ty
, s
));
757 let instance
= ty
::Instance
::mono(fx
.tcx
, def_id
).polymorphize(fx
.tcx
);
758 let func_ref
= fx
.get_function_ref(instance
);
759 let call
= fx
.bcx
.ins().call(func_ref
, &[llsize
, llalign
]);
760 let ptr
= fx
.bcx
.inst_results(call
)[0];
761 lval
.write_cvalue(fx
, CValue
::by_val(ptr
, box_layout
));
763 Rvalue
::NullaryOp(NullOp
::SizeOf
, ty
) => {
767 .is_sized(fx
.tcx
.at(stmt
.source_info
.span
), ParamEnv
::reveal_all()));
768 let ty_size
= fx
.layout_of(fx
.monomorphize(ty
)).size
.bytes();
770 CValue
::const_val(fx
, fx
.layout_of(fx
.tcx
.types
.usize), ty_size
.into());
771 lval
.write_cvalue(fx
, val
);
773 Rvalue
::Aggregate(ref kind
, ref operands
) => match kind
.as_ref() {
774 AggregateKind
::Array(_ty
) => {
775 for (i
, operand
) in operands
.iter().enumerate() {
776 let operand
= codegen_operand(fx
, operand
);
777 let index
= fx
.bcx
.ins().iconst(fx
.pointer_type
, i
as i64);
778 let to
= lval
.place_index(fx
, index
);
779 to
.write_cvalue(fx
, operand
);
782 _
=> unreachable
!("shouldn't exist at codegen {:?}", to_place_and_rval
.1),
786 StatementKind
::StorageLive(_
)
787 | StatementKind
::StorageDead(_
)
789 | StatementKind
::FakeRead(..)
790 | StatementKind
::Retag { .. }
791 | StatementKind
::AscribeUserType(..) => {}
793 StatementKind
::LlvmInlineAsm(asm
) => {
794 use rustc_span
::symbol
::Symbol
;
800 let rustc_hir
::LlvmInlineAsmInner
{
801 asm
: asm_code
, // Name
802 outputs
: output_names
, // Vec<LlvmInlineAsmOutput>
803 inputs
: input_names
, // Vec<Name>
804 clobbers
, // Vec<Name>
810 match asm_code
.as_str().trim() {
814 "mov %rbx, %rsi\n cpuid\n xchg %rbx, %rsi" => {
817 &[Symbol
::intern("{eax}"), Symbol
::intern("{ecx}")]
819 assert_eq
!(output_names
.len(), 4);
820 for (i
, c
) in (&["={eax}", "={esi}", "={ecx}", "={edx}"])
824 assert_eq
!(&output_names
[i
].constraint
.as_str(), c
);
825 assert
!(!output_names
[i
].is_rw
);
826 assert
!(!output_names
[i
].is_indirect
);
829 assert_eq
!(clobbers
, &[]);
832 assert
!(!alignstack
);
834 assert_eq
!(inputs
.len(), 2);
835 let leaf
= codegen_operand(fx
, &inputs
[0].1).load_scalar(fx
); // %eax
836 let subleaf
= codegen_operand(fx
, &inputs
[1].1).load_scalar(fx
); // %ecx
838 let (eax
, ebx
, ecx
, edx
) =
839 crate::intrinsics
::codegen_cpuid_call(fx
, leaf
, subleaf
);
841 assert_eq
!(outputs
.len(), 4);
842 codegen_place(fx
, outputs
[0])
843 .write_cvalue(fx
, CValue
::by_val(eax
, fx
.layout_of(fx
.tcx
.types
.u32)));
844 codegen_place(fx
, outputs
[1])
845 .write_cvalue(fx
, CValue
::by_val(ebx
, fx
.layout_of(fx
.tcx
.types
.u32)));
846 codegen_place(fx
, outputs
[2])
847 .write_cvalue(fx
, CValue
::by_val(ecx
, fx
.layout_of(fx
.tcx
.types
.u32)));
848 codegen_place(fx
, outputs
[3])
849 .write_cvalue(fx
, CValue
::by_val(edx
, fx
.layout_of(fx
.tcx
.types
.u32)));
852 assert_eq
!(input_names
, &[Symbol
::intern("{ecx}")]);
854 assert_eq
!(output_names
.len(), 2);
855 for (i
, c
) in (&["={eax}", "={edx}"]).iter().enumerate() {
856 assert_eq
!(&output_names
[i
].constraint
.as_str(), c
);
857 assert
!(!output_names
[i
].is_rw
);
858 assert
!(!output_names
[i
].is_indirect
);
861 assert_eq
!(clobbers
, &[]);
864 assert
!(!alignstack
);
866 crate::trap
::trap_unimplemented(fx
, "_xgetbv arch intrinsic is not supported");
868 // ___chkstk, ___chkstk_ms and __alloca are only used on Windows
871 .symbol_name(fx
.instance
)
873 .starts_with("___chkstk") =>
875 crate::trap
::trap_unimplemented(fx
, "Stack probes are not supported");
877 _
if fx
.tcx
.symbol_name(fx
.instance
).name
== "__alloca" => {
878 crate::trap
::trap_unimplemented(fx
, "Alloca is not supported");
880 // Used in sys::windows::abort_internal
882 crate::trap
::trap_unimplemented(fx
, "Windows abort");
887 .span_fatal(stmt
.source_info
.span
, "Inline assembly is not supported"),
890 StatementKind
::Coverage { .. }
=> fx
.tcx
.sess
.fatal("-Zcoverage is unimplemented"),
894 fn codegen_array_len
<'tcx
>(
895 fx
: &mut FunctionCx
<'_
, 'tcx
, impl Module
>,
898 match *place
.layout().ty
.kind() {
899 ty
::Array(_elem_ty
, len
) => {
902 .eval_usize(fx
.tcx
, ParamEnv
::reveal_all()) as i64;
903 fx
.bcx
.ins().iconst(fx
.pointer_type
, len
)
905 ty
::Slice(_elem_ty
) => place
906 .to_ptr_maybe_unsized()
908 .expect("Length metadata for slice place"),
909 _
=> bug
!("Rvalue::Len({:?})", place
),
913 pub(crate) fn codegen_place
<'tcx
>(
914 fx
: &mut FunctionCx
<'_
, 'tcx
, impl Module
>,
917 let mut cplace
= fx
.get_local_place(place
.local
);
919 for elem
in place
.projection
{
921 PlaceElem
::Deref
=> {
922 cplace
= cplace
.place_deref(fx
);
924 PlaceElem
::Field(field
, _ty
) => {
925 cplace
= cplace
.place_field(fx
, field
);
927 PlaceElem
::Index(local
) => {
928 let index
= fx
.get_local_place(local
).to_cvalue(fx
).load_scalar(fx
);
929 cplace
= cplace
.place_index(fx
, index
);
931 PlaceElem
::ConstantIndex
{
936 let offset
: u64 = offset
;
937 let index
= if !from_end
{
938 fx
.bcx
.ins().iconst(fx
.pointer_type
, offset
as i64)
940 let len
= codegen_array_len(fx
, cplace
);
941 fx
.bcx
.ins().iadd_imm(len
, -(offset
as i64))
943 cplace
= cplace
.place_index(fx
, index
);
945 PlaceElem
::Subslice { from, to, from_end }
=> {
946 // These indices are generated by slice patterns.
947 // slice[from:-to] in Python terms.
949 let from
: u64 = from
;
952 match cplace
.layout().ty
.kind() {
953 ty
::Array(elem_ty
, _len
) => {
954 assert
!(!from_end
, "array subslices are never `from_end`");
955 let elem_layout
= fx
.layout_of(elem_ty
);
956 let ptr
= cplace
.to_ptr();
957 cplace
= CPlace
::for_ptr(
958 ptr
.offset_i64(fx
, elem_layout
.size
.bytes() as i64 * (from
as i64)),
959 fx
.layout_of(fx
.tcx
.mk_array(elem_ty
, to
- from
)),
962 ty
::Slice(elem_ty
) => {
963 assert
!(from_end
, "slice subslices should be `from_end`");
964 let elem_layout
= fx
.layout_of(elem_ty
);
965 let (ptr
, len
) = cplace
.to_ptr_maybe_unsized();
966 let len
= len
.unwrap();
967 cplace
= CPlace
::for_ptr_with_extra(
968 ptr
.offset_i64(fx
, elem_layout
.size
.bytes() as i64 * (from
as i64)),
969 fx
.bcx
.ins().iadd_imm(len
, -(from
as i64 + to
as i64)),
976 PlaceElem
::Downcast(_adt_def
, variant
) => {
977 cplace
= cplace
.downcast_variant(fx
, variant
);
985 pub(crate) fn codegen_operand
<'tcx
>(
986 fx
: &mut FunctionCx
<'_
, 'tcx
, impl Module
>,
987 operand
: &Operand
<'tcx
>,
990 Operand
::Move(place
) | Operand
::Copy(place
) => {
991 let cplace
= codegen_place(fx
, *place
);
994 Operand
::Constant(const_
) => crate::constant
::codegen_constant(fx
, const_
),
998 pub(crate) fn codegen_panic
<'tcx
>(
999 fx
: &mut FunctionCx
<'_
, 'tcx
, impl Module
>,
1003 let location
= fx
.get_caller_location(span
).load_scalar(fx
);
1005 let msg_ptr
= fx
.anonymous_str("assert", msg_str
);
1009 .iconst(fx
.pointer_type
, i64::try_from(msg_str
.len()).unwrap());
1010 let args
= [msg_ptr
, msg_len
, location
];
1012 codegen_panic_inner(fx
, rustc_hir
::LangItem
::Panic
, &args
, span
);
1015 pub(crate) fn codegen_panic_inner
<'tcx
>(
1016 fx
: &mut FunctionCx
<'_
, 'tcx
, impl Module
>,
1017 lang_item
: rustc_hir
::LangItem
,
1025 .unwrap_or_else(|s
| fx
.tcx
.sess
.span_fatal(span
, &s
));
1027 let instance
= Instance
::mono(fx
.tcx
, def_id
).polymorphize(fx
.tcx
);
1028 let symbol_name
= fx
.tcx
.symbol_name(instance
).name
;
1032 vec
![fx
.pointer_type
, fx
.pointer_type
, fx
.pointer_type
],
1037 crate::trap
::trap_unreachable(fx
, "panic lang item returned");