1 //! Codegen of a single function
3 use rustc_ast
::InlineAsmOptions
;
4 use rustc_index
::vec
::IndexVec
;
5 use rustc_middle
::ty
::adjustment
::PointerCast
;
6 use rustc_middle
::ty
::layout
::FnAbiOf
;
7 use rustc_middle
::ty
::print
::with_no_trimmed_paths
;
9 use crate::constant
::ConstantCx
;
10 use crate::debuginfo
::FunctionDebugContext
;
11 use crate::prelude
::*;
12 use crate::pretty_clif
::CommentWriter
;
14 pub(crate) struct CodegenedFunction
{
18 clif_comments
: CommentWriter
,
19 func_debug_cx
: Option
<FunctionDebugContext
>,
22 #[cfg_attr(not(feature = "jit"), allow(dead_code))]
23 pub(crate) fn codegen_and_compile_fn
<'tcx
>(
25 cx
: &mut crate::CodegenCx
,
26 cached_context
: &mut Context
,
27 module
: &mut dyn Module
,
28 instance
: Instance
<'tcx
>,
31 crate::PrintOnPanic(|| format
!("{:?} {}", instance
, tcx
.symbol_name(instance
).name
));
33 let cached_func
= std
::mem
::replace(&mut cached_context
.func
, Function
::new());
34 let codegened_func
= codegen_fn(tcx
, cx
, cached_func
, module
, instance
);
36 compile_fn(cx
, cached_context
, module
, codegened_func
);
39 pub(crate) fn codegen_fn
<'tcx
>(
41 cx
: &mut crate::CodegenCx
,
42 cached_func
: Function
,
43 module
: &mut dyn Module
,
44 instance
: Instance
<'tcx
>,
45 ) -> CodegenedFunction
{
46 debug_assert
!(!instance
.substs
.needs_infer());
48 let mir
= tcx
.instance_mir(instance
.def
);
49 let _mir_guard
= crate::PrintOnPanic(|| {
50 let mut buf
= Vec
::new();
51 with_no_trimmed_paths
!({
52 rustc_middle
::mir
::pretty
::write_mir_fn(tcx
, mir
, &mut |_
, _
| Ok(()), &mut buf
)
55 String
::from_utf8_lossy(&buf
).into_owned()
59 let symbol_name
= tcx
.symbol_name(instance
).name
.to_string();
60 let sig
= get_function_sig(tcx
, module
.isa().triple(), instance
);
61 let func_id
= module
.declare_function(&symbol_name
, Linkage
::Local
, &sig
).unwrap();
63 // Make the FunctionBuilder
64 let mut func_ctx
= FunctionBuilderContext
::new();
65 let mut func
= cached_func
;
67 func
.name
= ExternalName
::user(0, func_id
.as_u32());
69 func
.collect_debug_info();
71 let mut bcx
= FunctionBuilder
::new(&mut func
, &mut func_ctx
);
74 let start_block
= bcx
.create_block();
75 let block_map
: IndexVec
<BasicBlock
, Block
> =
76 (0..mir
.basic_blocks
.len()).map(|_
| bcx
.create_block()).collect();
79 let target_config
= module
.target_config();
80 let pointer_type
= target_config
.pointer_type();
81 let clif_comments
= crate::pretty_clif
::CommentWriter
::new(tcx
, instance
);
83 let func_debug_cx
= if let Some(debug_context
) = &mut cx
.debug_context
{
84 Some(debug_context
.define_function(tcx
, &symbol_name
, mir
.span
))
89 let mut fx
= FunctionCx
{
95 constants_cx
: ConstantCx
::new(),
101 fn_abi
: Some(RevealAllLayoutCx(tcx
).fn_abi_of_instance(instance
, ty
::List
::empty())),
105 local_map
: IndexVec
::with_capacity(mir
.local_decls
.len()),
106 caller_location
: None
, // set by `codegen_fn_prelude`
109 last_source_file
: None
,
113 tcx
.sess
.time("codegen clif ir", || codegen_fn_body(&mut fx
, start_block
));
115 // Recover all necessary data from fx, before accessing func will prevent future access to it.
116 let symbol_name
= fx
.symbol_name
;
117 let clif_comments
= fx
.clif_comments
;
118 let func_debug_cx
= fx
.func_debug_cx
;
120 fx
.constants_cx
.finalize(fx
.tcx
, &mut *fx
.module
);
122 if cx
.should_write_ir
{
123 crate::pretty_clif
::write_clif_file(
124 tcx
.output_filenames(()),
134 verify_func(tcx
, &clif_comments
, &func
);
136 CodegenedFunction { symbol_name, func_id, func, clif_comments, func_debug_cx }
139 pub(crate) fn compile_fn(
140 cx
: &mut crate::CodegenCx
,
141 cached_context
: &mut Context
,
142 module
: &mut dyn Module
,
143 codegened_func
: CodegenedFunction
,
145 let clif_comments
= codegened_func
.clif_comments
;
147 // Store function in context
148 let context
= cached_context
;
150 context
.func
= codegened_func
.func
;
152 // If the return block is not reachable, then the SSA builder may have inserted an `iconst.i128`
153 // instruction, which doesn't have an encoding.
154 context
.compute_cfg();
155 context
.compute_domtree();
156 context
.eliminate_unreachable_code(module
.isa()).unwrap();
157 context
.dce(module
.isa()).unwrap();
158 // Some Cranelift optimizations expect the domtree to not yet be computed and as such don't
159 // invalidate it when it would change.
160 context
.domtree
.clear();
162 #[cfg(any())] // This is never true
166 let func_clone
= context
.func
.clone();
167 let clif_comments_clone
= clif_comments
.clone();
168 let mut clif
= String
::new();
169 for flag
in module
.isa().flags().iter() {
170 writeln
!(clif
, "set {}", flag
).unwrap();
172 write
!(clif
, "target {}", module
.isa().triple().architecture
.to_string()).unwrap();
173 for isa_flag
in module
.isa().isa_flags().iter() {
174 write
!(clif
, " {}", isa_flag
).unwrap();
176 writeln
!(clif
, "\n").unwrap();
177 crate::PrintOnPanic(move || {
178 let mut clif
= clif
.clone();
179 ::cranelift_codegen
::write
::decorate_function(
180 &mut &clif_comments_clone
,
190 cx
.profiler
.verbose_generic_activity("define function").run(|| {
191 context
.want_disasm
= cx
.should_write_ir
;
192 module
.define_function(codegened_func
.func_id
, context
).unwrap();
195 if cx
.should_write_ir
{
196 // Write optimized function to file for debugging
197 crate::pretty_clif
::write_clif_file(
198 &cx
.output_filenames
,
199 &codegened_func
.symbol_name
,
206 if let Some(disasm
) = &context
.compiled_code().unwrap().disasm
{
207 crate::pretty_clif
::write_ir_file(
208 &cx
.output_filenames
,
209 &format
!("{}.vcode", codegened_func
.symbol_name
),
210 |file
| file
.write_all(disasm
.as_bytes()),
215 // Define debuginfo for function
216 let isa
= module
.isa();
217 let debug_context
= &mut cx
.debug_context
;
218 let unwind_context
= &mut cx
.unwind_context
;
219 cx
.profiler
.verbose_generic_activity("generate debug info").run(|| {
220 if let Some(debug_context
) = debug_context
{
221 codegened_func
.func_debug_cx
.unwrap().finalize(
223 codegened_func
.func_id
,
227 unwind_context
.add_function(codegened_func
.func_id
, &context
, isa
);
231 pub(crate) fn verify_func(
233 writer
: &crate::pretty_clif
::CommentWriter
,
236 tcx
.sess
.time("verify clif ir", || {
237 let flags
= cranelift_codegen
::settings
::Flags
::new(cranelift_codegen
::settings
::builder());
238 match cranelift_codegen
::verify_function(&func
, &flags
) {
241 tcx
.sess
.err(&format
!("{:?}", err
));
242 let pretty_error
= cranelift_codegen
::print_errors
::pretty_verifier_error(
244 Some(Box
::new(writer
)),
247 tcx
.sess
.fatal(&format
!("cranelift verify error:\n{}", pretty_error
));
253 fn codegen_fn_body(fx
: &mut FunctionCx
<'_
, '_
, '_
>, start_block
: Block
) {
254 if !crate::constant
::check_constants(fx
) {
255 fx
.bcx
.append_block_params_for_function_params(fx
.block_map
[START_BLOCK
]);
256 fx
.bcx
.switch_to_block(fx
.block_map
[START_BLOCK
]);
257 // compilation should have been aborted
258 fx
.bcx
.ins().trap(TrapCode
::UnreachableCodeReached
);
262 let arg_uninhabited
= fx
265 .any(|arg
| fx
.layout_of(fx
.monomorphize(fx
.mir
.local_decls
[arg
].ty
)).abi
.is_uninhabited());
267 fx
.bcx
.append_block_params_for_function_params(fx
.block_map
[START_BLOCK
]);
268 fx
.bcx
.switch_to_block(fx
.block_map
[START_BLOCK
]);
269 fx
.bcx
.ins().trap(TrapCode
::UnreachableCodeReached
);
272 fx
.tcx
.sess
.time("codegen prelude", || crate::abi
::codegen_fn_prelude(fx
, start_block
));
274 for (bb
, bb_data
) in fx
.mir
.basic_blocks
.iter_enumerated() {
275 let block
= fx
.get_block(bb
);
276 fx
.bcx
.switch_to_block(block
);
278 if bb_data
.is_cleanup
{
279 // Unwinding after panicking is not supported
282 // FIXME Once unwinding is supported and Cranelift supports marking blocks as cold, do
283 // so for cleanup blocks.
287 for stmt
in &bb_data
.statements
{
288 fx
.set_debug_loc(stmt
.source_info
);
289 codegen_stmt(fx
, block
, stmt
);
292 if fx
.clif_comments
.enabled() {
293 let mut terminator_head
= "\n".to_string();
294 with_no_trimmed_paths
!({
295 bb_data
.terminator().kind
.fmt_head(&mut terminator_head
).unwrap();
297 let inst
= fx
.bcx
.func
.layout
.last_inst(block
).unwrap();
298 fx
.add_comment(inst
, terminator_head
);
301 let source_info
= bb_data
.terminator().source_info
;
302 fx
.set_debug_loc(source_info
);
304 match &bb_data
.terminator().kind
{
305 TerminatorKind
::Goto { target }
=> {
306 if let TerminatorKind
::Return
= fx
.mir
[*target
].terminator().kind
{
307 let mut can_immediately_return
= true;
308 for stmt
in &fx
.mir
[*target
].statements
{
309 if let StatementKind
::StorageDead(_
) = stmt
.kind
{
311 // FIXME Can sometimes happen, see rust-lang/rust#70531
312 can_immediately_return
= false;
317 if can_immediately_return
{
318 crate::abi
::codegen_return(fx
);
323 let block
= fx
.get_block(*target
);
324 fx
.bcx
.ins().jump(block
, &[]);
326 TerminatorKind
::Return
=> {
327 crate::abi
::codegen_return(fx
);
329 TerminatorKind
::Assert { cond, expected, msg, target, cleanup: _ }
=> {
330 if !fx
.tcx
.sess
.overflow_checks() {
331 if let mir
::AssertKind
::OverflowNeg(_
) = *msg
{
332 let target
= fx
.get_block(*target
);
333 fx
.bcx
.ins().jump(target
, &[]);
337 let cond
= codegen_operand(fx
, cond
).load_scalar(fx
);
339 let target
= fx
.get_block(*target
);
340 let failure
= fx
.bcx
.create_block();
341 fx
.bcx
.set_cold_block(failure
);
344 fx
.bcx
.ins().brz(cond
, failure
, &[]);
346 fx
.bcx
.ins().brnz(cond
, failure
, &[]);
348 fx
.bcx
.ins().jump(target
, &[]);
350 fx
.bcx
.switch_to_block(failure
);
354 AssertKind
::BoundsCheck { ref len, ref index }
=> {
355 let len
= codegen_operand(fx
, len
).load_scalar(fx
);
356 let index
= codegen_operand(fx
, index
).load_scalar(fx
);
357 let location
= fx
.get_caller_location(source_info
).load_scalar(fx
);
361 rustc_hir
::LangItem
::PanicBoundsCheck
,
362 &[index
, len
, location
],
367 let msg_str
= msg
.description();
368 codegen_panic(fx
, msg_str
, source_info
);
373 TerminatorKind
::SwitchInt { discr, switch_ty, targets }
=> {
374 let discr
= codegen_operand(fx
, discr
).load_scalar(fx
);
376 let use_bool_opt
= switch_ty
.kind() == fx
.tcx
.types
.bool
.kind()
377 || (targets
.iter().count() == 1 && targets
.iter().next().unwrap().0 == 0);
379 assert_eq
!(targets
.iter().count(), 1);
380 let (then_value
, then_block
) = targets
.iter().next().unwrap();
381 let then_block
= fx
.get_block(then_block
);
382 let else_block
= fx
.get_block(targets
.otherwise());
383 let test_zero
= match then_value
{
386 _
=> unreachable
!("{:?}", targets
),
389 let discr
= crate::optimize
::peephole
::maybe_unwrap_bint(&mut fx
.bcx
, discr
);
390 let (discr
, is_inverted
) =
391 crate::optimize
::peephole
::maybe_unwrap_bool_not(&mut fx
.bcx
, discr
);
392 let test_zero
= if is_inverted { !test_zero }
else { test_zero }
;
393 let discr
= crate::optimize
::peephole
::maybe_unwrap_bint(&mut fx
.bcx
, discr
);
394 if let Some(taken
) = crate::optimize
::peephole
::maybe_known_branch_taken(
395 &fx
.bcx
, discr
, test_zero
,
398 fx
.bcx
.ins().jump(then_block
, &[]);
400 fx
.bcx
.ins().jump(else_block
, &[]);
404 fx
.bcx
.ins().brz(discr
, then_block
, &[]);
405 fx
.bcx
.ins().jump(else_block
, &[]);
407 fx
.bcx
.ins().brnz(discr
, then_block
, &[]);
408 fx
.bcx
.ins().jump(else_block
, &[]);
412 let mut switch
= ::cranelift_frontend
::Switch
::new();
413 for (value
, block
) in targets
.iter() {
414 let block
= fx
.get_block(block
);
415 switch
.set_entry(value
, block
);
417 let otherwise_block
= fx
.get_block(targets
.otherwise());
418 switch
.emit(&mut fx
.bcx
, discr
, otherwise_block
);
421 TerminatorKind
::Call
{
430 fx
.tcx
.sess
.time("codegen call", || {
431 crate::abi
::codegen_terminator_call(
433 mir
::SourceInfo { span: *fn_span, ..source_info }
,
441 TerminatorKind
::InlineAsm
{
449 if options
.contains(InlineAsmOptions
::MAY_UNWIND
) {
450 fx
.tcx
.sess
.span_fatal(
452 "cranelift doesn't support unwinding from inline assembly.",
456 crate::inline_asm
::codegen_inline_asm(
465 TerminatorKind
::Resume
| TerminatorKind
::Abort
=> {
466 // FIXME implement unwinding
467 fx
.bcx
.ins().trap(TrapCode
::UnreachableCodeReached
);
469 TerminatorKind
::Unreachable
=> {
470 fx
.bcx
.ins().trap(TrapCode
::UnreachableCodeReached
);
472 TerminatorKind
::Yield { .. }
473 | TerminatorKind
::FalseEdge { .. }
474 | TerminatorKind
::FalseUnwind { .. }
475 | TerminatorKind
::DropAndReplace { .. }
476 | TerminatorKind
::GeneratorDrop
=> {
477 bug
!("shouldn't exist at codegen {:?}", bb_data
.terminator());
479 TerminatorKind
::Drop { place, target, unwind: _ }
=> {
480 let drop_place
= codegen_place(fx
, *place
);
481 crate::abi
::codegen_drop(fx
, source_info
, drop_place
);
483 let target_block
= fx
.get_block(*target
);
484 fx
.bcx
.ins().jump(target_block
, &[]);
489 fx
.bcx
.seal_all_blocks();
493 fn codegen_stmt
<'tcx
>(
494 fx
: &mut FunctionCx
<'_
, '_
, 'tcx
>,
495 #[allow(unused_variables)] cur_block: Block,
496 stmt
: &Statement
<'tcx
>,
498 let _print_guard
= crate::PrintOnPanic(|| format
!("stmt {:?}", stmt
));
500 fx
.set_debug_loc(stmt
.source_info
);
502 #[cfg(any())] // This is never true
504 StatementKind
::StorageLive(..) | StatementKind
::StorageDead(..) => {}
// Those are not very useful
506 if fx
.clif_comments
.enabled() {
507 let inst
= fx
.bcx
.func
.layout
.last_inst(cur_block
).unwrap();
508 fx
.add_comment(inst
, format
!("{:?}", stmt
));
514 StatementKind
::SetDiscriminant { place, variant_index }
=> {
515 let place
= codegen_place(fx
, **place
);
516 crate::discriminant
::codegen_set_discriminant(fx
, place
, *variant_index
);
518 StatementKind
::Assign(to_place_and_rval
) => {
519 let lval
= codegen_place(fx
, to_place_and_rval
.0);
520 let dest_layout
= lval
.layout();
521 match to_place_and_rval
.1 {
522 Rvalue
::Use(ref operand
) => {
523 let val
= codegen_operand(fx
, operand
);
524 lval
.write_cvalue(fx
, val
);
526 Rvalue
::CopyForDeref(place
) => {
527 let cplace
= codegen_place(fx
, place
);
528 let val
= cplace
.to_cvalue(fx
);
529 lval
.write_cvalue(fx
, val
)
531 Rvalue
::Ref(_
, _
, place
) | Rvalue
::AddressOf(_
, place
) => {
532 let place
= codegen_place(fx
, place
);
533 let ref_
= place
.place_ref(fx
, lval
.layout());
534 lval
.write_cvalue(fx
, ref_
);
536 Rvalue
::ThreadLocalRef(def_id
) => {
537 let val
= crate::constant
::codegen_tls_ref(fx
, def_id
, lval
.layout());
538 lval
.write_cvalue(fx
, val
);
540 Rvalue
::BinaryOp(bin_op
, ref lhs_rhs
) => {
541 let lhs
= codegen_operand(fx
, &lhs_rhs
.0);
542 let rhs
= codegen_operand(fx
, &lhs_rhs
.1);
544 let res
= crate::num
::codegen_binop(fx
, bin_op
, lhs
, rhs
);
545 lval
.write_cvalue(fx
, res
);
547 Rvalue
::CheckedBinaryOp(bin_op
, ref lhs_rhs
) => {
548 let lhs
= codegen_operand(fx
, &lhs_rhs
.0);
549 let rhs
= codegen_operand(fx
, &lhs_rhs
.1);
551 let res
= if !fx
.tcx
.sess
.overflow_checks() {
553 crate::num
::codegen_int_binop(fx
, bin_op
, lhs
, rhs
).load_scalar(fx
);
554 let is_overflow
= fx
.bcx
.ins().iconst(types
::I8
, 0);
555 CValue
::by_val_pair(val
, is_overflow
, lval
.layout())
557 crate::num
::codegen_checked_int_binop(fx
, bin_op
, lhs
, rhs
)
560 lval
.write_cvalue(fx
, res
);
562 Rvalue
::UnaryOp(un_op
, ref operand
) => {
563 let operand
= codegen_operand(fx
, operand
);
564 let layout
= operand
.layout();
565 let val
= operand
.load_scalar(fx
);
566 let res
= match un_op
{
567 UnOp
::Not
=> match layout
.ty
.kind() {
569 let res
= fx
.bcx
.ins().icmp_imm(IntCC
::Equal
, val
, 0);
570 CValue
::by_val(fx
.bcx
.ins().bint(types
::I8
, res
), layout
)
572 ty
::Uint(_
) | ty
::Int(_
) => {
573 CValue
::by_val(fx
.bcx
.ins().bnot(val
), layout
)
575 _
=> unreachable
!("un op Not for {:?}", layout
.ty
),
577 UnOp
::Neg
=> match layout
.ty
.kind() {
578 ty
::Int(IntTy
::I128
) => {
579 // FIXME remove this case once ineg.i128 works
581 CValue
::const_val(fx
, layout
, ty
::ScalarInt
::null(layout
.size
));
582 crate::num
::codegen_int_binop(fx
, BinOp
::Sub
, zero
, operand
)
584 ty
::Int(_
) => CValue
::by_val(fx
.bcx
.ins().ineg(val
), layout
),
585 ty
::Float(_
) => CValue
::by_val(fx
.bcx
.ins().fneg(val
), layout
),
586 _
=> unreachable
!("un op Neg for {:?}", layout
.ty
),
589 lval
.write_cvalue(fx
, res
);
592 CastKind
::Pointer(PointerCast
::ReifyFnPointer
),
596 let from_ty
= fx
.monomorphize(operand
.ty(&fx
.mir
.local_decls
, fx
.tcx
));
597 let to_layout
= fx
.layout_of(fx
.monomorphize(to_ty
));
598 match *from_ty
.kind() {
599 ty
::FnDef(def_id
, substs
) => {
600 let func_ref
= fx
.get_function_ref(
601 Instance
::resolve_for_fn_ptr(
603 ParamEnv
::reveal_all(),
608 .polymorphize(fx
.tcx
),
610 let func_addr
= fx
.bcx
.ins().func_addr(fx
.pointer_type
, func_ref
);
611 lval
.write_cvalue(fx
, CValue
::by_val(func_addr
, to_layout
));
613 _
=> bug
!("Trying to ReifyFnPointer on non FnDef {:?}", from_ty
),
617 CastKind
::Pointer(PointerCast
::UnsafeFnPointer
),
622 CastKind
::Pointer(PointerCast
::MutToConstPointer
),
627 CastKind
::Pointer(PointerCast
::ArrayToPointer
),
631 let to_layout
= fx
.layout_of(fx
.monomorphize(to_ty
));
632 let operand
= codegen_operand(fx
, operand
);
633 lval
.write_cvalue(fx
, operand
.cast_pointer_to(to_layout
));
637 | CastKind
::PointerExposeAddress
638 | CastKind
::PointerFromExposedAddress
,
642 let operand
= codegen_operand(fx
, operand
);
643 let from_ty
= operand
.layout().ty
;
644 let to_ty
= fx
.monomorphize(to_ty
);
646 fn is_fat_ptr
<'tcx
>(fx
: &FunctionCx
<'_
, '_
, 'tcx
>, ty
: Ty
<'tcx
>) -> bool
{
647 ty
.builtin_deref(true)
648 .map(|ty
::TypeAndMut { ty: pointee_ty, mutbl: _ }
| {
649 has_ptr_meta(fx
.tcx
, pointee_ty
)
654 if is_fat_ptr(fx
, from_ty
) {
655 if is_fat_ptr(fx
, to_ty
) {
656 // fat-ptr -> fat-ptr
657 lval
.write_cvalue(fx
, operand
.cast_pointer_to(dest_layout
));
659 // fat-ptr -> thin-ptr
660 let (ptr
, _extra
) = operand
.load_scalar_pair(fx
);
661 lval
.write_cvalue(fx
, CValue
::by_val(ptr
, dest_layout
))
664 let to_clif_ty
= fx
.clif_type(to_ty
).unwrap();
665 let from
= operand
.load_scalar(fx
);
667 let res
= clif_int_or_float_cast(
674 lval
.write_cvalue(fx
, CValue
::by_val(res
, dest_layout
));
678 CastKind
::Pointer(PointerCast
::ClosureFnPointer(_
)),
682 let operand
= codegen_operand(fx
, operand
);
683 match *operand
.layout().ty
.kind() {
684 ty
::Closure(def_id
, substs
) => {
685 let instance
= Instance
::resolve_closure(
689 ty
::ClosureKind
::FnOnce
,
691 .expect("failed to normalize and resolve closure during codegen")
692 .polymorphize(fx
.tcx
);
693 let func_ref
= fx
.get_function_ref(instance
);
694 let func_addr
= fx
.bcx
.ins().func_addr(fx
.pointer_type
, func_ref
);
695 lval
.write_cvalue(fx
, CValue
::by_val(func_addr
, lval
.layout()));
697 _
=> bug
!("{} cannot be cast to a fn ptr", operand
.layout().ty
),
700 Rvalue
::Cast(CastKind
::Pointer(PointerCast
::Unsize
), ref operand
, _to_ty
) => {
701 let operand
= codegen_operand(fx
, operand
);
702 operand
.unsize_value(fx
, lval
);
704 Rvalue
::Cast(CastKind
::DynStar
, _
, _
) => {
708 Rvalue
::Discriminant(place
) => {
709 let place
= codegen_place(fx
, place
);
710 let value
= place
.to_cvalue(fx
);
711 crate::discriminant
::codegen_get_discriminant(fx
, lval
, value
, dest_layout
);
713 Rvalue
::Repeat(ref operand
, times
) => {
714 let operand
= codegen_operand(fx
, operand
);
717 .eval(fx
.tcx
, ParamEnv
::reveal_all())
719 .try_to_bits(fx
.tcx
.data_layout
.pointer_size
)
721 if operand
.layout().size
.bytes() == 0 {
722 // Do nothing for ZST's
723 } else if fx
.clif_type(operand
.layout().ty
) == Some(types
::I8
) {
724 let times
= fx
.bcx
.ins().iconst(fx
.pointer_type
, times
as i64);
725 // FIXME use emit_small_memset where possible
726 let addr
= lval
.to_ptr().get_addr(fx
);
727 let val
= operand
.load_scalar(fx
);
728 fx
.bcx
.call_memset(fx
.target_config
, addr
, val
, times
);
730 let loop_block
= fx
.bcx
.create_block();
731 let loop_block2
= fx
.bcx
.create_block();
732 let done_block
= fx
.bcx
.create_block();
733 let index
= fx
.bcx
.append_block_param(loop_block
, fx
.pointer_type
);
734 let zero
= fx
.bcx
.ins().iconst(fx
.pointer_type
, 0);
735 fx
.bcx
.ins().jump(loop_block
, &[zero
]);
737 fx
.bcx
.switch_to_block(loop_block
);
738 let done
= fx
.bcx
.ins().icmp_imm(IntCC
::Equal
, index
, times
as i64);
739 fx
.bcx
.ins().brnz(done
, done_block
, &[]);
740 fx
.bcx
.ins().jump(loop_block2
, &[]);
742 fx
.bcx
.switch_to_block(loop_block2
);
743 let to
= lval
.place_index(fx
, index
);
744 to
.write_cvalue(fx
, operand
);
745 let index
= fx
.bcx
.ins().iadd_imm(index
, 1);
746 fx
.bcx
.ins().jump(loop_block
, &[index
]);
748 fx
.bcx
.switch_to_block(done_block
);
752 Rvalue
::Len(place
) => {
753 let place
= codegen_place(fx
, place
);
754 let usize_layout
= fx
.layout_of(fx
.tcx
.types
.usize);
755 let len
= codegen_array_len(fx
, place
);
756 lval
.write_cvalue(fx
, CValue
::by_val(len
, usize_layout
));
758 Rvalue
::ShallowInitBox(ref operand
, content_ty
) => {
759 let content_ty
= fx
.monomorphize(content_ty
);
760 let box_layout
= fx
.layout_of(fx
.tcx
.mk_box(content_ty
));
761 let operand
= codegen_operand(fx
, operand
);
762 let operand
= operand
.load_scalar(fx
);
763 lval
.write_cvalue(fx
, CValue
::by_val(operand
, box_layout
));
765 Rvalue
::NullaryOp(null_op
, ty
) => {
769 .is_sized(fx
.tcx
.at(stmt
.source_info
.span
), ParamEnv
::reveal_all())
771 let layout
= fx
.layout_of(fx
.monomorphize(ty
));
772 let val
= match null_op
{
773 NullOp
::SizeOf
=> layout
.size
.bytes(),
774 NullOp
::AlignOf
=> layout
.align
.abi
.bytes(),
776 let val
= CValue
::const_val(fx
, fx
.layout_of(fx
.tcx
.types
.usize), val
.into());
777 lval
.write_cvalue(fx
, val
);
779 Rvalue
::Aggregate(ref kind
, ref operands
) => match kind
.as_ref() {
780 AggregateKind
::Array(_ty
) => {
781 for (i
, operand
) in operands
.iter().enumerate() {
782 let operand
= codegen_operand(fx
, operand
);
783 let index
= fx
.bcx
.ins().iconst(fx
.pointer_type
, i
as i64);
784 let to
= lval
.place_index(fx
, index
);
785 to
.write_cvalue(fx
, operand
);
788 _
=> unreachable
!("shouldn't exist at codegen {:?}", to_place_and_rval
.1),
792 StatementKind
::StorageLive(_
)
793 | StatementKind
::StorageDead(_
)
794 | StatementKind
::Deinit(_
)
796 | StatementKind
::FakeRead(..)
797 | StatementKind
::Retag { .. }
798 | StatementKind
::AscribeUserType(..) => {}
800 StatementKind
::Coverage { .. }
=> fx
.tcx
.sess
.fatal("-Zcoverage is unimplemented"),
801 StatementKind
::Intrinsic(ref intrinsic
) => match &**intrinsic
{
802 // We ignore `assume` intrinsics, they are only useful for optimizations
803 NonDivergingIntrinsic
::Assume(_
) => {}
804 NonDivergingIntrinsic
::CopyNonOverlapping(mir
::CopyNonOverlapping
{
809 let dst
= codegen_operand(fx
, dst
);
812 .pointee_info_at(fx
, rustc_target
::abi
::Size
::ZERO
)
813 .expect("Expected pointer");
814 let dst
= dst
.load_scalar(fx
);
815 let src
= codegen_operand(fx
, src
).load_scalar(fx
);
816 let count
= codegen_operand(fx
, count
).load_scalar(fx
);
817 let elem_size
: u64 = pointee
.size
.bytes();
818 let bytes
= if elem_size
!= 1 {
819 fx
.bcx
.ins().imul_imm(count
, elem_size
as i64)
823 fx
.bcx
.call_memcpy(fx
.target_config
, dst
, src
, bytes
);
829 fn codegen_array_len
<'tcx
>(fx
: &mut FunctionCx
<'_
, '_
, 'tcx
>, place
: CPlace
<'tcx
>) -> Value
{
830 match *place
.layout().ty
.kind() {
831 ty
::Array(_elem_ty
, len
) => {
832 let len
= fx
.monomorphize(len
).eval_usize(fx
.tcx
, ParamEnv
::reveal_all()) as i64;
833 fx
.bcx
.ins().iconst(fx
.pointer_type
, len
)
835 ty
::Slice(_elem_ty
) => {
836 place
.to_ptr_maybe_unsized().1.expect("Length metadata for slice place")
838 _
=> bug
!("Rvalue::Len({:?})", place
),
842 pub(crate) fn codegen_place
<'tcx
>(
843 fx
: &mut FunctionCx
<'_
, '_
, 'tcx
>,
846 let mut cplace
= fx
.get_local_place(place
.local
);
848 for elem
in place
.projection
{
850 PlaceElem
::Deref
=> {
851 cplace
= cplace
.place_deref(fx
);
853 PlaceElem
::Field(field
, _ty
) => {
854 cplace
= cplace
.place_field(fx
, field
);
856 PlaceElem
::Index(local
) => {
857 let index
= fx
.get_local_place(local
).to_cvalue(fx
).load_scalar(fx
);
858 cplace
= cplace
.place_index(fx
, index
);
860 PlaceElem
::ConstantIndex { offset, min_length: _, from_end }
=> {
861 let offset
: u64 = offset
;
862 let index
= if !from_end
{
863 fx
.bcx
.ins().iconst(fx
.pointer_type
, offset
as i64)
865 let len
= codegen_array_len(fx
, cplace
);
866 fx
.bcx
.ins().iadd_imm(len
, -(offset
as i64))
868 cplace
= cplace
.place_index(fx
, index
);
870 PlaceElem
::Subslice { from, to, from_end }
=> {
871 // These indices are generated by slice patterns.
872 // slice[from:-to] in Python terms.
874 let from
: u64 = from
;
877 match cplace
.layout().ty
.kind() {
878 ty
::Array(elem_ty
, _len
) => {
879 assert
!(!from_end
, "array subslices are never `from_end`");
880 let elem_layout
= fx
.layout_of(*elem_ty
);
881 let ptr
= cplace
.to_ptr();
882 cplace
= CPlace
::for_ptr(
883 ptr
.offset_i64(fx
, elem_layout
.size
.bytes() as i64 * (from
as i64)),
884 fx
.layout_of(fx
.tcx
.mk_array(*elem_ty
, to
- from
)),
887 ty
::Slice(elem_ty
) => {
888 assert
!(from_end
, "slice subslices should be `from_end`");
889 let elem_layout
= fx
.layout_of(*elem_ty
);
890 let (ptr
, len
) = cplace
.to_ptr_maybe_unsized();
891 let len
= len
.unwrap();
892 cplace
= CPlace
::for_ptr_with_extra(
893 ptr
.offset_i64(fx
, elem_layout
.size
.bytes() as i64 * (from
as i64)),
894 fx
.bcx
.ins().iadd_imm(len
, -(from
as i64 + to
as i64)),
901 PlaceElem
::Downcast(_adt_def
, variant
) => {
902 cplace
= cplace
.downcast_variant(fx
, variant
);
910 pub(crate) fn codegen_operand
<'tcx
>(
911 fx
: &mut FunctionCx
<'_
, '_
, 'tcx
>,
912 operand
: &Operand
<'tcx
>,
915 Operand
::Move(place
) | Operand
::Copy(place
) => {
916 let cplace
= codegen_place(fx
, *place
);
919 Operand
::Constant(const_
) => crate::constant
::codegen_constant(fx
, const_
),
923 pub(crate) fn codegen_panic
<'tcx
>(
924 fx
: &mut FunctionCx
<'_
, '_
, 'tcx
>,
926 source_info
: mir
::SourceInfo
,
928 let location
= fx
.get_caller_location(source_info
).load_scalar(fx
);
930 let msg_ptr
= fx
.anonymous_str(msg_str
);
931 let msg_len
= fx
.bcx
.ins().iconst(fx
.pointer_type
, i64::try_from(msg_str
.len()).unwrap());
932 let args
= [msg_ptr
, msg_len
, location
];
934 codegen_panic_inner(fx
, rustc_hir
::LangItem
::Panic
, &args
, source_info
.span
);
937 pub(crate) fn codegen_panic_inner
<'tcx
>(
938 fx
: &mut FunctionCx
<'_
, '_
, 'tcx
>,
939 lang_item
: rustc_hir
::LangItem
,
947 .unwrap_or_else(|e
| fx
.tcx
.sess
.span_fatal(span
, e
.to_string()));
949 let instance
= Instance
::mono(fx
.tcx
, def_id
).polymorphize(fx
.tcx
);
950 let symbol_name
= fx
.tcx
.symbol_name(instance
).name
;
955 AbiParam
::new(fx
.pointer_type
),
956 AbiParam
::new(fx
.pointer_type
),
957 AbiParam
::new(fx
.pointer_type
),
963 fx
.bcx
.ins().trap(TrapCode
::UnreachableCodeReached
);