1 //! Codegen of a single function
3 use rustc_ast
::InlineAsmOptions
;
4 use rustc_index
::vec
::IndexVec
;
5 use rustc_middle
::ty
::adjustment
::PointerCast
;
6 use rustc_middle
::ty
::layout
::FnAbiOf
;
7 use rustc_middle
::ty
::print
::with_no_trimmed_paths
;
9 use cranelift_codegen
::ir
::UserFuncName
;
11 use crate::constant
::ConstantCx
;
12 use crate::debuginfo
::FunctionDebugContext
;
13 use crate::prelude
::*;
14 use crate::pretty_clif
::CommentWriter
;
16 pub(crate) struct CodegenedFunction
{
20 clif_comments
: CommentWriter
,
21 func_debug_cx
: Option
<FunctionDebugContext
>,
24 pub(crate) fn codegen_fn
<'tcx
>(
26 cx
: &mut crate::CodegenCx
,
27 cached_func
: Function
,
28 module
: &mut dyn Module
,
29 instance
: Instance
<'tcx
>,
30 ) -> CodegenedFunction
{
31 debug_assert
!(!instance
.substs
.needs_infer());
33 let symbol_name
= tcx
.symbol_name(instance
).name
.to_string();
34 let _timer
= tcx
.prof
.generic_activity_with_arg("codegen fn", &*symbol_name
);
36 let mir
= tcx
.instance_mir(instance
.def
);
37 let _mir_guard
= crate::PrintOnPanic(|| {
38 let mut buf
= Vec
::new();
39 with_no_trimmed_paths
!({
40 rustc_middle
::mir
::pretty
::write_mir_fn(tcx
, mir
, &mut |_
, _
| Ok(()), &mut buf
)
43 String
::from_utf8_lossy(&buf
).into_owned()
47 let sig
= get_function_sig(tcx
, module
.target_config().default_call_conv
, instance
);
48 let func_id
= module
.declare_function(&symbol_name
, Linkage
::Local
, &sig
).unwrap();
50 // Make the FunctionBuilder
51 let mut func_ctx
= FunctionBuilderContext
::new();
52 let mut func
= cached_func
;
54 func
.name
= UserFuncName
::user(0, func_id
.as_u32());
56 func
.collect_debug_info();
58 let mut bcx
= FunctionBuilder
::new(&mut func
, &mut func_ctx
);
61 let start_block
= bcx
.create_block();
62 let block_map
: IndexVec
<BasicBlock
, Block
> =
63 (0..mir
.basic_blocks
.len()).map(|_
| bcx
.create_block()).collect();
66 let target_config
= module
.target_config();
67 let pointer_type
= target_config
.pointer_type();
68 let clif_comments
= crate::pretty_clif
::CommentWriter
::new(tcx
, instance
);
70 let func_debug_cx
= if let Some(debug_context
) = &mut cx
.debug_context
{
71 Some(debug_context
.define_function(tcx
, &symbol_name
, mir
.span
))
76 let mut fx
= FunctionCx
{
82 constants_cx
: ConstantCx
::new(),
88 fn_abi
: Some(RevealAllLayoutCx(tcx
).fn_abi_of_instance(instance
, ty
::List
::empty())),
92 local_map
: IndexVec
::with_capacity(mir
.local_decls
.len()),
93 caller_location
: None
, // set by `codegen_fn_prelude`
96 last_source_file
: None
,
100 tcx
.prof
.generic_activity("codegen clif ir").run(|| codegen_fn_body(&mut fx
, start_block
));
101 fx
.bcx
.seal_all_blocks();
104 // Recover all necessary data from fx, before accessing func will prevent future access to it.
105 let symbol_name
= fx
.symbol_name
;
106 let clif_comments
= fx
.clif_comments
;
107 let func_debug_cx
= fx
.func_debug_cx
;
109 fx
.constants_cx
.finalize(fx
.tcx
, &mut *fx
.module
);
111 if cx
.should_write_ir
{
112 crate::pretty_clif
::write_clif_file(
113 tcx
.output_filenames(()),
123 verify_func(tcx
, &clif_comments
, &func
);
125 CodegenedFunction { symbol_name, func_id, func, clif_comments, func_debug_cx }
128 pub(crate) fn compile_fn(
129 cx
: &mut crate::CodegenCx
,
130 cached_context
: &mut Context
,
131 module
: &mut dyn Module
,
132 codegened_func
: CodegenedFunction
,
135 cx
.profiler
.generic_activity_with_arg("compile function", &*codegened_func
.symbol_name
);
137 let clif_comments
= codegened_func
.clif_comments
;
139 // Store function in context
140 let context
= cached_context
;
142 context
.func
= codegened_func
.func
;
144 // If the return block is not reachable, then the SSA builder may have inserted an `iconst.i128`
145 // instruction, which doesn't have an encoding.
146 context
.compute_cfg();
147 context
.compute_domtree();
148 context
.eliminate_unreachable_code(module
.isa()).unwrap();
149 context
.dce(module
.isa()).unwrap();
150 // Some Cranelift optimizations expect the domtree to not yet be computed and as such don't
151 // invalidate it when it would change.
152 context
.domtree
.clear();
154 #[cfg(any())] // This is never true
158 let func_clone
= context
.func
.clone();
159 let clif_comments_clone
= clif_comments
.clone();
160 let mut clif
= String
::new();
161 for flag
in module
.isa().flags().iter() {
162 writeln
!(clif
, "set {}", flag
).unwrap();
164 write
!(clif
, "target {}", module
.isa().triple().architecture
.to_string()).unwrap();
165 for isa_flag
in module
.isa().isa_flags().iter() {
166 write
!(clif
, " {}", isa_flag
).unwrap();
168 writeln
!(clif
, "\n").unwrap();
169 crate::PrintOnPanic(move || {
170 let mut clif
= clif
.clone();
171 ::cranelift_codegen
::write
::decorate_function(
172 &mut &clif_comments_clone
,
182 cx
.profiler
.generic_activity("define function").run(|| {
183 context
.want_disasm
= cx
.should_write_ir
;
184 module
.define_function(codegened_func
.func_id
, context
).unwrap();
186 if cx
.profiler
.enabled() {
187 let mut recording_args
= false;
189 .generic_activity_with_arg_recorder(
190 "define function (clif pass timings)",
192 let pass_times
= cranelift_codegen
::timing
::take_current();
193 // Replace newlines with | as measureme doesn't allow control characters like
194 // newlines inside strings.
195 recorder
.record_arg(format
!("{}", pass_times
).replace("\n", " | "));
196 recording_args
= true;
201 // Wait a tiny bit to ensure chrome's profiler doesn't hide the event
202 std
::thread
::sleep(std
::time
::Duration
::from_nanos(2))
208 if cx
.should_write_ir
{
209 // Write optimized function to file for debugging
210 crate::pretty_clif
::write_clif_file(
211 &cx
.output_filenames
,
212 &codegened_func
.symbol_name
,
219 if let Some(disasm
) = &context
.compiled_code().unwrap().disasm
{
220 crate::pretty_clif
::write_ir_file(
221 &cx
.output_filenames
,
222 &format
!("{}.vcode", codegened_func
.symbol_name
),
223 |file
| file
.write_all(disasm
.as_bytes()),
228 // Define debuginfo for function
229 let isa
= module
.isa();
230 let debug_context
= &mut cx
.debug_context
;
231 let unwind_context
= &mut cx
.unwind_context
;
232 cx
.profiler
.generic_activity("generate debug info").run(|| {
233 if let Some(debug_context
) = debug_context
{
234 codegened_func
.func_debug_cx
.unwrap().finalize(
236 codegened_func
.func_id
,
240 unwind_context
.add_function(codegened_func
.func_id
, &context
, isa
);
244 pub(crate) fn verify_func(
246 writer
: &crate::pretty_clif
::CommentWriter
,
249 tcx
.prof
.generic_activity("verify clif ir").run(|| {
250 let flags
= cranelift_codegen
::settings
::Flags
::new(cranelift_codegen
::settings
::builder());
251 match cranelift_codegen
::verify_function(&func
, &flags
) {
254 tcx
.sess
.err(&format
!("{:?}", err
));
255 let pretty_error
= cranelift_codegen
::print_errors
::pretty_verifier_error(
257 Some(Box
::new(writer
)),
260 tcx
.sess
.fatal(&format
!("cranelift verify error:\n{}", pretty_error
));
266 fn codegen_fn_body(fx
: &mut FunctionCx
<'_
, '_
, '_
>, start_block
: Block
) {
267 if !crate::constant
::check_constants(fx
) {
268 fx
.bcx
.append_block_params_for_function_params(fx
.block_map
[START_BLOCK
]);
269 fx
.bcx
.switch_to_block(fx
.block_map
[START_BLOCK
]);
270 // compilation should have been aborted
271 fx
.bcx
.ins().trap(TrapCode
::UnreachableCodeReached
);
275 let arg_uninhabited
= fx
278 .any(|arg
| fx
.layout_of(fx
.monomorphize(fx
.mir
.local_decls
[arg
].ty
)).abi
.is_uninhabited());
280 fx
.bcx
.append_block_params_for_function_params(fx
.block_map
[START_BLOCK
]);
281 fx
.bcx
.switch_to_block(fx
.block_map
[START_BLOCK
]);
282 fx
.bcx
.ins().trap(TrapCode
::UnreachableCodeReached
);
287 .generic_activity("codegen prelude")
288 .run(|| crate::abi
::codegen_fn_prelude(fx
, start_block
));
290 for (bb
, bb_data
) in fx
.mir
.basic_blocks
.iter_enumerated() {
291 let block
= fx
.get_block(bb
);
292 fx
.bcx
.switch_to_block(block
);
294 if bb_data
.is_cleanup
{
295 // Unwinding after panicking is not supported
298 // FIXME Once unwinding is supported and Cranelift supports marking blocks as cold, do
299 // so for cleanup blocks.
303 for stmt
in &bb_data
.statements
{
304 fx
.set_debug_loc(stmt
.source_info
);
305 codegen_stmt(fx
, block
, stmt
);
308 if fx
.clif_comments
.enabled() {
309 let mut terminator_head
= "\n".to_string();
310 with_no_trimmed_paths
!({
311 bb_data
.terminator().kind
.fmt_head(&mut terminator_head
).unwrap();
313 let inst
= fx
.bcx
.func
.layout
.last_inst(block
).unwrap();
314 fx
.add_comment(inst
, terminator_head
);
317 let source_info
= bb_data
.terminator().source_info
;
318 fx
.set_debug_loc(source_info
);
321 crate::PrintOnPanic(|| format
!("terminator {:?}", bb_data
.terminator().kind
));
323 match &bb_data
.terminator().kind
{
324 TerminatorKind
::Goto { target }
=> {
325 if let TerminatorKind
::Return
= fx
.mir
[*target
].terminator().kind
{
326 let mut can_immediately_return
= true;
327 for stmt
in &fx
.mir
[*target
].statements
{
328 if let StatementKind
::StorageDead(_
) = stmt
.kind
{
330 // FIXME Can sometimes happen, see rust-lang/rust#70531
331 can_immediately_return
= false;
336 if can_immediately_return
{
337 crate::abi
::codegen_return(fx
);
342 let block
= fx
.get_block(*target
);
343 fx
.bcx
.ins().jump(block
, &[]);
345 TerminatorKind
::Return
=> {
346 crate::abi
::codegen_return(fx
);
348 TerminatorKind
::Assert { cond, expected, msg, target, cleanup: _ }
=> {
349 if !fx
.tcx
.sess
.overflow_checks() {
350 let overflow_not_to_check
= match msg
{
351 AssertKind
::OverflowNeg(..) => true,
352 AssertKind
::Overflow(op
, ..) => op
.is_checkable(),
355 if overflow_not_to_check
{
356 let target
= fx
.get_block(*target
);
357 fx
.bcx
.ins().jump(target
, &[]);
361 let cond
= codegen_operand(fx
, cond
).load_scalar(fx
);
363 let target
= fx
.get_block(*target
);
364 let failure
= fx
.bcx
.create_block();
365 fx
.bcx
.set_cold_block(failure
);
368 fx
.bcx
.ins().brz(cond
, failure
, &[]);
370 fx
.bcx
.ins().brnz(cond
, failure
, &[]);
372 fx
.bcx
.ins().jump(target
, &[]);
374 fx
.bcx
.switch_to_block(failure
);
378 AssertKind
::BoundsCheck { ref len, ref index }
=> {
379 let len
= codegen_operand(fx
, len
).load_scalar(fx
);
380 let index
= codegen_operand(fx
, index
).load_scalar(fx
);
381 let location
= fx
.get_caller_location(source_info
).load_scalar(fx
);
385 rustc_hir
::LangItem
::PanicBoundsCheck
,
386 &[index
, len
, location
],
391 let msg_str
= msg
.description();
392 codegen_panic(fx
, msg_str
, source_info
);
397 TerminatorKind
::SwitchInt { discr, targets }
=> {
398 let discr
= codegen_operand(fx
, discr
);
399 let switch_ty
= discr
.layout().ty
;
400 let discr
= discr
.load_scalar(fx
);
402 let use_bool_opt
= switch_ty
.kind() == fx
.tcx
.types
.bool
.kind()
403 || (targets
.iter().count() == 1 && targets
.iter().next().unwrap().0 == 0);
405 assert_eq
!(targets
.iter().count(), 1);
406 let (then_value
, then_block
) = targets
.iter().next().unwrap();
407 let then_block
= fx
.get_block(then_block
);
408 let else_block
= fx
.get_block(targets
.otherwise());
409 let test_zero
= match then_value
{
412 _
=> unreachable
!("{:?}", targets
),
415 let (discr
, is_inverted
) =
416 crate::optimize
::peephole
::maybe_unwrap_bool_not(&mut fx
.bcx
, discr
);
417 let test_zero
= if is_inverted { !test_zero }
else { test_zero }
;
418 if let Some(taken
) = crate::optimize
::peephole
::maybe_known_branch_taken(
419 &fx
.bcx
, discr
, test_zero
,
422 fx
.bcx
.ins().jump(then_block
, &[]);
424 fx
.bcx
.ins().jump(else_block
, &[]);
428 fx
.bcx
.ins().brz(discr
, then_block
, &[]);
429 fx
.bcx
.ins().jump(else_block
, &[]);
431 fx
.bcx
.ins().brnz(discr
, then_block
, &[]);
432 fx
.bcx
.ins().jump(else_block
, &[]);
436 let mut switch
= ::cranelift_frontend
::Switch
::new();
437 for (value
, block
) in targets
.iter() {
438 let block
= fx
.get_block(block
);
439 switch
.set_entry(value
, block
);
441 let otherwise_block
= fx
.get_block(targets
.otherwise());
442 switch
.emit(&mut fx
.bcx
, discr
, otherwise_block
);
445 TerminatorKind
::Call
{
454 fx
.tcx
.prof
.generic_activity("codegen call").run(|| {
455 crate::abi
::codegen_terminator_call(
457 mir
::SourceInfo { span: *fn_span, ..source_info }
,
465 TerminatorKind
::InlineAsm
{
473 if options
.contains(InlineAsmOptions
::MAY_UNWIND
) {
474 fx
.tcx
.sess
.span_fatal(
476 "cranelift doesn't support unwinding from inline assembly.",
480 crate::inline_asm
::codegen_inline_asm(
489 TerminatorKind
::Abort
=> {
490 codegen_panic_cannot_unwind(fx
, source_info
);
492 TerminatorKind
::Resume
=> {
493 // FIXME implement unwinding
494 fx
.bcx
.ins().trap(TrapCode
::UnreachableCodeReached
);
496 TerminatorKind
::Unreachable
=> {
497 fx
.bcx
.ins().trap(TrapCode
::UnreachableCodeReached
);
499 TerminatorKind
::Yield { .. }
500 | TerminatorKind
::FalseEdge { .. }
501 | TerminatorKind
::FalseUnwind { .. }
502 | TerminatorKind
::DropAndReplace { .. }
503 | TerminatorKind
::GeneratorDrop
=> {
504 bug
!("shouldn't exist at codegen {:?}", bb_data
.terminator());
506 TerminatorKind
::Drop { place, target, unwind: _ }
=> {
507 let drop_place
= codegen_place(fx
, *place
);
508 crate::abi
::codegen_drop(fx
, source_info
, drop_place
);
510 let target_block
= fx
.get_block(*target
);
511 fx
.bcx
.ins().jump(target_block
, &[]);
517 fn codegen_stmt
<'tcx
>(
518 fx
: &mut FunctionCx
<'_
, '_
, 'tcx
>,
519 #[allow(unused_variables)] cur_block: Block,
520 stmt
: &Statement
<'tcx
>,
522 let _print_guard
= crate::PrintOnPanic(|| format
!("stmt {:?}", stmt
));
524 fx
.set_debug_loc(stmt
.source_info
);
526 #[cfg(any())] // This is never true
528 StatementKind
::StorageLive(..) | StatementKind
::StorageDead(..) => {}
// Those are not very useful
530 if fx
.clif_comments
.enabled() {
531 let inst
= fx
.bcx
.func
.layout
.last_inst(cur_block
).unwrap();
532 fx
.add_comment(inst
, format
!("{:?}", stmt
));
538 StatementKind
::SetDiscriminant { place, variant_index }
=> {
539 let place
= codegen_place(fx
, **place
);
540 crate::discriminant
::codegen_set_discriminant(fx
, place
, *variant_index
);
542 StatementKind
::Assign(to_place_and_rval
) => {
543 let lval
= codegen_place(fx
, to_place_and_rval
.0);
544 let dest_layout
= lval
.layout();
545 match to_place_and_rval
.1 {
546 Rvalue
::Use(ref operand
) => {
547 let val
= codegen_operand(fx
, operand
);
548 lval
.write_cvalue(fx
, val
);
550 Rvalue
::CopyForDeref(place
) => {
551 let cplace
= codegen_place(fx
, place
);
552 let val
= cplace
.to_cvalue(fx
);
553 lval
.write_cvalue(fx
, val
)
555 Rvalue
::Ref(_
, _
, place
) | Rvalue
::AddressOf(_
, place
) => {
556 let place
= codegen_place(fx
, place
);
557 let ref_
= place
.place_ref(fx
, lval
.layout());
558 lval
.write_cvalue(fx
, ref_
);
560 Rvalue
::ThreadLocalRef(def_id
) => {
561 let val
= crate::constant
::codegen_tls_ref(fx
, def_id
, lval
.layout());
562 lval
.write_cvalue(fx
, val
);
564 Rvalue
::BinaryOp(bin_op
, ref lhs_rhs
) => {
565 let lhs
= codegen_operand(fx
, &lhs_rhs
.0);
566 let rhs
= codegen_operand(fx
, &lhs_rhs
.1);
568 let res
= crate::num
::codegen_binop(fx
, bin_op
, lhs
, rhs
);
569 lval
.write_cvalue(fx
, res
);
571 Rvalue
::CheckedBinaryOp(bin_op
, ref lhs_rhs
) => {
572 let lhs
= codegen_operand(fx
, &lhs_rhs
.0);
573 let rhs
= codegen_operand(fx
, &lhs_rhs
.1);
575 let res
= crate::num
::codegen_checked_int_binop(fx
, bin_op
, lhs
, rhs
);
576 lval
.write_cvalue(fx
, res
);
578 Rvalue
::UnaryOp(un_op
, ref operand
) => {
579 let operand
= codegen_operand(fx
, operand
);
580 let layout
= operand
.layout();
581 let val
= operand
.load_scalar(fx
);
582 let res
= match un_op
{
583 UnOp
::Not
=> match layout
.ty
.kind() {
585 let res
= fx
.bcx
.ins().icmp_imm(IntCC
::Equal
, val
, 0);
586 CValue
::by_val(res
, layout
)
588 ty
::Uint(_
) | ty
::Int(_
) => {
589 CValue
::by_val(fx
.bcx
.ins().bnot(val
), layout
)
591 _
=> unreachable
!("un op Not for {:?}", layout
.ty
),
593 UnOp
::Neg
=> match layout
.ty
.kind() {
594 ty
::Int(_
) => CValue
::by_val(fx
.bcx
.ins().ineg(val
), layout
),
595 ty
::Float(_
) => CValue
::by_val(fx
.bcx
.ins().fneg(val
), layout
),
596 _
=> unreachable
!("un op Neg for {:?}", layout
.ty
),
599 lval
.write_cvalue(fx
, res
);
602 CastKind
::Pointer(PointerCast
::ReifyFnPointer
),
606 let from_ty
= fx
.monomorphize(operand
.ty(&fx
.mir
.local_decls
, fx
.tcx
));
607 let to_layout
= fx
.layout_of(fx
.monomorphize(to_ty
));
608 match *from_ty
.kind() {
609 ty
::FnDef(def_id
, substs
) => {
610 let func_ref
= fx
.get_function_ref(
611 Instance
::resolve_for_fn_ptr(
613 ParamEnv
::reveal_all(),
618 .polymorphize(fx
.tcx
),
620 let func_addr
= fx
.bcx
.ins().func_addr(fx
.pointer_type
, func_ref
);
621 lval
.write_cvalue(fx
, CValue
::by_val(func_addr
, to_layout
));
623 _
=> bug
!("Trying to ReifyFnPointer on non FnDef {:?}", from_ty
),
627 CastKind
::Pointer(PointerCast
::UnsafeFnPointer
),
632 CastKind
::Pointer(PointerCast
::MutToConstPointer
),
637 CastKind
::Pointer(PointerCast
::ArrayToPointer
),
641 let to_layout
= fx
.layout_of(fx
.monomorphize(to_ty
));
642 let operand
= codegen_operand(fx
, operand
);
643 lval
.write_cvalue(fx
, operand
.cast_pointer_to(to_layout
));
647 | CastKind
::FloatToFloat
648 | CastKind
::FloatToInt
649 | CastKind
::IntToFloat
650 | CastKind
::FnPtrToPtr
652 | CastKind
::PointerExposeAddress
653 | CastKind
::PointerFromExposedAddress
,
657 let operand
= codegen_operand(fx
, operand
);
658 let from_ty
= operand
.layout().ty
;
659 let to_ty
= fx
.monomorphize(to_ty
);
661 fn is_fat_ptr
<'tcx
>(fx
: &FunctionCx
<'_
, '_
, 'tcx
>, ty
: Ty
<'tcx
>) -> bool
{
662 ty
.builtin_deref(true)
663 .map(|ty
::TypeAndMut { ty: pointee_ty, mutbl: _ }
| {
664 has_ptr_meta(fx
.tcx
, pointee_ty
)
669 if is_fat_ptr(fx
, from_ty
) {
670 if is_fat_ptr(fx
, to_ty
) {
671 // fat-ptr -> fat-ptr
672 lval
.write_cvalue(fx
, operand
.cast_pointer_to(dest_layout
));
674 // fat-ptr -> thin-ptr
675 let (ptr
, _extra
) = operand
.load_scalar_pair(fx
);
676 lval
.write_cvalue(fx
, CValue
::by_val(ptr
, dest_layout
))
679 let to_clif_ty
= fx
.clif_type(to_ty
).unwrap();
680 let from
= operand
.load_scalar(fx
);
682 let res
= clif_int_or_float_cast(
689 lval
.write_cvalue(fx
, CValue
::by_val(res
, dest_layout
));
693 CastKind
::Pointer(PointerCast
::ClosureFnPointer(_
)),
697 let operand
= codegen_operand(fx
, operand
);
698 match *operand
.layout().ty
.kind() {
699 ty
::Closure(def_id
, substs
) => {
700 let instance
= Instance
::resolve_closure(
704 ty
::ClosureKind
::FnOnce
,
706 .expect("failed to normalize and resolve closure during codegen")
707 .polymorphize(fx
.tcx
);
708 let func_ref
= fx
.get_function_ref(instance
);
709 let func_addr
= fx
.bcx
.ins().func_addr(fx
.pointer_type
, func_ref
);
710 lval
.write_cvalue(fx
, CValue
::by_val(func_addr
, lval
.layout()));
712 _
=> bug
!("{} cannot be cast to a fn ptr", operand
.layout().ty
),
715 Rvalue
::Cast(CastKind
::Pointer(PointerCast
::Unsize
), ref operand
, _to_ty
) => {
716 let operand
= codegen_operand(fx
, operand
);
717 operand
.unsize_value(fx
, lval
);
719 Rvalue
::Cast(CastKind
::DynStar
, ref operand
, _
) => {
720 let operand
= codegen_operand(fx
, operand
);
721 operand
.coerce_dyn_star(fx
, lval
);
723 Rvalue
::Discriminant(place
) => {
724 let place
= codegen_place(fx
, place
);
725 let value
= place
.to_cvalue(fx
);
726 crate::discriminant
::codegen_get_discriminant(fx
, lval
, value
, dest_layout
);
728 Rvalue
::Repeat(ref operand
, times
) => {
729 let operand
= codegen_operand(fx
, operand
);
732 .eval(fx
.tcx
, ParamEnv
::reveal_all())
734 .try_to_bits(fx
.tcx
.data_layout
.pointer_size
)
736 if operand
.layout().size
.bytes() == 0 {
737 // Do nothing for ZST's
738 } else if fx
.clif_type(operand
.layout().ty
) == Some(types
::I8
) {
739 let times
= fx
.bcx
.ins().iconst(fx
.pointer_type
, times
as i64);
740 // FIXME use emit_small_memset where possible
741 let addr
= lval
.to_ptr().get_addr(fx
);
742 let val
= operand
.load_scalar(fx
);
743 fx
.bcx
.call_memset(fx
.target_config
, addr
, val
, times
);
745 let loop_block
= fx
.bcx
.create_block();
746 let loop_block2
= fx
.bcx
.create_block();
747 let done_block
= fx
.bcx
.create_block();
748 let index
= fx
.bcx
.append_block_param(loop_block
, fx
.pointer_type
);
749 let zero
= fx
.bcx
.ins().iconst(fx
.pointer_type
, 0);
750 fx
.bcx
.ins().jump(loop_block
, &[zero
]);
752 fx
.bcx
.switch_to_block(loop_block
);
753 let done
= fx
.bcx
.ins().icmp_imm(IntCC
::Equal
, index
, times
as i64);
754 fx
.bcx
.ins().brnz(done
, done_block
, &[]);
755 fx
.bcx
.ins().jump(loop_block2
, &[]);
757 fx
.bcx
.switch_to_block(loop_block2
);
758 let to
= lval
.place_index(fx
, index
);
759 to
.write_cvalue(fx
, operand
);
760 let index
= fx
.bcx
.ins().iadd_imm(index
, 1);
761 fx
.bcx
.ins().jump(loop_block
, &[index
]);
763 fx
.bcx
.switch_to_block(done_block
);
767 Rvalue
::Len(place
) => {
768 let place
= codegen_place(fx
, place
);
769 let usize_layout
= fx
.layout_of(fx
.tcx
.types
.usize);
770 let len
= codegen_array_len(fx
, place
);
771 lval
.write_cvalue(fx
, CValue
::by_val(len
, usize_layout
));
773 Rvalue
::ShallowInitBox(ref operand
, content_ty
) => {
774 let content_ty
= fx
.monomorphize(content_ty
);
775 let box_layout
= fx
.layout_of(fx
.tcx
.mk_box(content_ty
));
776 let operand
= codegen_operand(fx
, operand
);
777 let operand
= operand
.load_scalar(fx
);
778 lval
.write_cvalue(fx
, CValue
::by_val(operand
, box_layout
));
780 Rvalue
::NullaryOp(null_op
, ty
) => {
781 assert
!(lval
.layout().ty
.is_sized(fx
.tcx
, ParamEnv
::reveal_all()));
782 let layout
= fx
.layout_of(fx
.monomorphize(ty
));
783 let val
= match null_op
{
784 NullOp
::SizeOf
=> layout
.size
.bytes(),
785 NullOp
::AlignOf
=> layout
.align
.abi
.bytes(),
787 let val
= CValue
::const_val(fx
, fx
.layout_of(fx
.tcx
.types
.usize), val
.into());
788 lval
.write_cvalue(fx
, val
);
790 Rvalue
::Aggregate(ref kind
, ref operands
) => {
791 let (variant_index
, variant_dest
, active_field_index
) = match **kind
{
792 mir
::AggregateKind
::Adt(_
, variant_index
, _
, _
, active_field_index
) => {
793 let variant_dest
= lval
.downcast_variant(fx
, variant_index
);
794 (variant_index
, variant_dest
, active_field_index
)
796 _
=> (VariantIdx
::from_u32(0), lval
, None
),
798 if active_field_index
.is_some() {
799 assert_eq
!(operands
.len(), 1);
801 for (i
, operand
) in operands
.iter().enumerate() {
802 let operand
= codegen_operand(fx
, operand
);
803 let field_index
= active_field_index
.unwrap_or(i
);
804 let to
= if let mir
::AggregateKind
::Array(_
) = **kind
{
805 let index
= fx
.bcx
.ins().iconst(fx
.pointer_type
, field_index
as i64);
806 variant_dest
.place_index(fx
, index
)
808 variant_dest
.place_field(fx
, mir
::Field
::new(field_index
))
810 to
.write_cvalue(fx
, operand
);
812 crate::discriminant
::codegen_set_discriminant(fx
, lval
, variant_index
);
816 StatementKind
::StorageLive(_
)
817 | StatementKind
::StorageDead(_
)
818 | StatementKind
::Deinit(_
)
819 | StatementKind
::ConstEvalCounter
821 | StatementKind
::FakeRead(..)
822 | StatementKind
::Retag { .. }
823 | StatementKind
::AscribeUserType(..) => {}
825 StatementKind
::Coverage { .. }
=> fx
.tcx
.sess
.fatal("-Zcoverage is unimplemented"),
826 StatementKind
::Intrinsic(ref intrinsic
) => match &**intrinsic
{
827 // We ignore `assume` intrinsics, they are only useful for optimizations
828 NonDivergingIntrinsic
::Assume(_
) => {}
829 NonDivergingIntrinsic
::CopyNonOverlapping(mir
::CopyNonOverlapping
{
834 let dst
= codegen_operand(fx
, dst
);
837 .pointee_info_at(fx
, rustc_target
::abi
::Size
::ZERO
)
838 .expect("Expected pointer");
839 let dst
= dst
.load_scalar(fx
);
840 let src
= codegen_operand(fx
, src
).load_scalar(fx
);
841 let count
= codegen_operand(fx
, count
).load_scalar(fx
);
842 let elem_size
: u64 = pointee
.size
.bytes();
843 let bytes
= if elem_size
!= 1 {
844 fx
.bcx
.ins().imul_imm(count
, elem_size
as i64)
848 fx
.bcx
.call_memcpy(fx
.target_config
, dst
, src
, bytes
);
854 fn codegen_array_len
<'tcx
>(fx
: &mut FunctionCx
<'_
, '_
, 'tcx
>, place
: CPlace
<'tcx
>) -> Value
{
855 match *place
.layout().ty
.kind() {
856 ty
::Array(_elem_ty
, len
) => {
857 let len
= fx
.monomorphize(len
).eval_target_usize(fx
.tcx
, ParamEnv
::reveal_all()) as i64;
858 fx
.bcx
.ins().iconst(fx
.pointer_type
, len
)
860 ty
::Slice(_elem_ty
) => {
861 place
.to_ptr_maybe_unsized().1.expect("Length metadata for slice place")
863 _
=> bug
!("Rvalue::Len({:?})", place
),
867 pub(crate) fn codegen_place
<'tcx
>(
868 fx
: &mut FunctionCx
<'_
, '_
, 'tcx
>,
871 let mut cplace
= fx
.get_local_place(place
.local
);
873 for elem
in place
.projection
{
875 PlaceElem
::Deref
=> {
876 cplace
= cplace
.place_deref(fx
);
878 PlaceElem
::OpaqueCast(ty
) => cplace
= cplace
.place_opaque_cast(fx
, ty
),
879 PlaceElem
::Field(field
, _ty
) => {
880 cplace
= cplace
.place_field(fx
, field
);
882 PlaceElem
::Index(local
) => {
883 let index
= fx
.get_local_place(local
).to_cvalue(fx
).load_scalar(fx
);
884 cplace
= cplace
.place_index(fx
, index
);
886 PlaceElem
::ConstantIndex { offset, min_length: _, from_end }
=> {
887 let offset
: u64 = offset
;
888 let index
= if !from_end
{
889 fx
.bcx
.ins().iconst(fx
.pointer_type
, offset
as i64)
891 let len
= codegen_array_len(fx
, cplace
);
892 fx
.bcx
.ins().iadd_imm(len
, -(offset
as i64))
894 cplace
= cplace
.place_index(fx
, index
);
896 PlaceElem
::Subslice { from, to, from_end }
=> {
897 // These indices are generated by slice patterns.
898 // slice[from:-to] in Python terms.
900 let from
: u64 = from
;
903 match cplace
.layout().ty
.kind() {
904 ty
::Array(elem_ty
, _len
) => {
905 assert
!(!from_end
, "array subslices are never `from_end`");
906 let elem_layout
= fx
.layout_of(*elem_ty
);
907 let ptr
= cplace
.to_ptr();
908 cplace
= CPlace
::for_ptr(
909 ptr
.offset_i64(fx
, elem_layout
.size
.bytes() as i64 * (from
as i64)),
910 fx
.layout_of(fx
.tcx
.mk_array(*elem_ty
, to
- from
)),
913 ty
::Slice(elem_ty
) => {
914 assert
!(from_end
, "slice subslices should be `from_end`");
915 let elem_layout
= fx
.layout_of(*elem_ty
);
916 let (ptr
, len
) = cplace
.to_ptr_maybe_unsized();
917 let len
= len
.unwrap();
918 cplace
= CPlace
::for_ptr_with_extra(
919 ptr
.offset_i64(fx
, elem_layout
.size
.bytes() as i64 * (from
as i64)),
920 fx
.bcx
.ins().iadd_imm(len
, -(from
as i64 + to
as i64)),
927 PlaceElem
::Downcast(_adt_def
, variant
) => {
928 cplace
= cplace
.downcast_variant(fx
, variant
);
936 pub(crate) fn codegen_operand
<'tcx
>(
937 fx
: &mut FunctionCx
<'_
, '_
, 'tcx
>,
938 operand
: &Operand
<'tcx
>,
941 Operand
::Move(place
) | Operand
::Copy(place
) => {
942 let cplace
= codegen_place(fx
, *place
);
945 Operand
::Constant(const_
) => crate::constant
::codegen_constant_operand(fx
, const_
),
949 pub(crate) fn codegen_panic
<'tcx
>(
950 fx
: &mut FunctionCx
<'_
, '_
, 'tcx
>,
952 source_info
: mir
::SourceInfo
,
954 let location
= fx
.get_caller_location(source_info
).load_scalar(fx
);
956 let msg_ptr
= fx
.anonymous_str(msg_str
);
957 let msg_len
= fx
.bcx
.ins().iconst(fx
.pointer_type
, i64::try_from(msg_str
.len()).unwrap());
958 let args
= [msg_ptr
, msg_len
, location
];
960 codegen_panic_inner(fx
, rustc_hir
::LangItem
::Panic
, &args
, source_info
.span
);
963 pub(crate) fn codegen_panic_nounwind
<'tcx
>(
964 fx
: &mut FunctionCx
<'_
, '_
, 'tcx
>,
966 source_info
: mir
::SourceInfo
,
968 let msg_ptr
= fx
.anonymous_str(msg_str
);
969 let msg_len
= fx
.bcx
.ins().iconst(fx
.pointer_type
, i64::try_from(msg_str
.len()).unwrap());
970 let args
= [msg_ptr
, msg_len
];
972 codegen_panic_inner(fx
, rustc_hir
::LangItem
::PanicNounwind
, &args
, source_info
.span
);
975 pub(crate) fn codegen_panic_cannot_unwind
<'tcx
>(
976 fx
: &mut FunctionCx
<'_
, '_
, 'tcx
>,
977 source_info
: mir
::SourceInfo
,
981 codegen_panic_inner(fx
, rustc_hir
::LangItem
::PanicCannotUnwind
, &args
, source_info
.span
);
984 fn codegen_panic_inner
<'tcx
>(
985 fx
: &mut FunctionCx
<'_
, '_
, 'tcx
>,
986 lang_item
: rustc_hir
::LangItem
,
994 .unwrap_or_else(|e
| fx
.tcx
.sess
.span_fatal(span
, e
.to_string()));
996 let instance
= Instance
::mono(fx
.tcx
, def_id
).polymorphize(fx
.tcx
);
997 let symbol_name
= fx
.tcx
.symbol_name(instance
).name
;
1001 args
.iter().map(|&arg
| AbiParam
::new(fx
.bcx
.func
.dfg
.value_type(arg
))).collect(),
1006 fx
.bcx
.ins().trap(TrapCode
::UnreachableCodeReached
);