]>
Commit | Line | Data |
---|---|---|
29967ef6 XL |
1 | //! Codegen of a single function |
2 | ||
3 | use rustc_index::vec::IndexVec; | |
4 | use rustc_middle::ty::adjustment::PointerCast; | |
5869c6ff XL |
5 | use rustc_middle::ty::layout::FnAbiExt; |
6 | use rustc_target::abi::call::FnAbi; | |
29967ef6 XL |
7 | |
8 | use crate::prelude::*; | |
9 | ||
10 | pub(crate) fn codegen_fn<'tcx>( | |
6a06907d | 11 | cx: &mut crate::CodegenCx<'_, 'tcx>, |
29967ef6 XL |
12 | instance: Instance<'tcx>, |
13 | linkage: Linkage, | |
14 | ) { | |
15 | let tcx = cx.tcx; | |
16 | ||
fc512014 XL |
17 | let _inst_guard = |
18 | crate::PrintOnPanic(|| format!("{:?} {}", instance, tcx.symbol_name(instance).name)); | |
19 | debug_assert!(!instance.substs.needs_infer()); | |
20 | ||
29967ef6 XL |
21 | let mir = tcx.instance_mir(instance.def); |
22 | ||
23 | // Declare function | |
5869c6ff XL |
24 | let name = tcx.symbol_name(instance).name.to_string(); |
25 | let sig = get_function_sig(tcx, cx.module.isa().triple(), instance); | |
29967ef6 XL |
26 | let func_id = cx.module.declare_function(&name, linkage, &sig).unwrap(); |
27 | ||
28 | cx.cached_context.clear(); | |
29 | ||
30 | // Make the FunctionBuilder | |
31 | let mut func_ctx = FunctionBuilderContext::new(); | |
32 | let mut func = std::mem::replace(&mut cx.cached_context.func, Function::new()); | |
33 | func.name = ExternalName::user(0, func_id.as_u32()); | |
34 | func.signature = sig; | |
35 | func.collect_debug_info(); | |
36 | ||
37 | let mut bcx = FunctionBuilder::new(&mut func, &mut func_ctx); | |
38 | ||
39 | // Predefine blocks | |
40 | let start_block = bcx.create_block(); | |
6a06907d XL |
41 | let block_map: IndexVec<BasicBlock, Block> = |
42 | (0..mir.basic_blocks().len()).map(|_| bcx.create_block()).collect(); | |
29967ef6 XL |
43 | |
44 | // Make FunctionCx | |
45 | let pointer_type = cx.module.target_config().pointer_type(); | |
46 | let clif_comments = crate::pretty_clif::CommentWriter::new(tcx, instance); | |
47 | ||
48 | let mut fx = FunctionCx { | |
49 | cx, | |
50 | tcx, | |
51 | pointer_type, | |
52 | ||
53 | instance, | |
54 | mir, | |
5869c6ff | 55 | fn_abi: Some(FnAbi::of_instance(&RevealAllLayoutCx(tcx), instance, &[])), |
29967ef6 XL |
56 | |
57 | bcx, | |
58 | block_map, | |
59 | local_map: IndexVec::with_capacity(mir.local_decls.len()), | |
60 | caller_location: None, // set by `codegen_fn_prelude` | |
61 | cold_blocks: EntitySet::new(), | |
62 | ||
63 | clif_comments, | |
64 | source_info_set: indexmap::IndexSet::new(), | |
65 | next_ssa_var: 0, | |
66 | ||
67 | inline_asm_index: 0, | |
68 | }; | |
69 | ||
6a06907d XL |
70 | let arg_uninhabited = fx |
71 | .mir | |
72 | .args_iter() | |
73 | .any(|arg| fx.layout_of(fx.monomorphize(&fx.mir.local_decls[arg].ty)).abi.is_uninhabited()); | |
29967ef6 | 74 | |
6a06907d XL |
75 | if !crate::constant::check_constants(&mut fx) { |
76 | fx.bcx.append_block_params_for_function_params(fx.block_map[START_BLOCK]); | |
77 | fx.bcx.switch_to_block(fx.block_map[START_BLOCK]); | |
78 | crate::trap::trap_unreachable(&mut fx, "compilation should have been aborted"); | |
79 | } else if arg_uninhabited { | |
80 | fx.bcx.append_block_params_for_function_params(fx.block_map[START_BLOCK]); | |
29967ef6 XL |
81 | fx.bcx.switch_to_block(fx.block_map[START_BLOCK]); |
82 | crate::trap::trap_unreachable(&mut fx, "function has uninhabited argument"); | |
83 | } else { | |
84 | tcx.sess.time("codegen clif ir", || { | |
6a06907d XL |
85 | tcx.sess |
86 | .time("codegen prelude", || crate::abi::codegen_fn_prelude(&mut fx, start_block)); | |
29967ef6 XL |
87 | codegen_fn_content(&mut fx); |
88 | }); | |
89 | } | |
90 | ||
91 | // Recover all necessary data from fx, before accessing func will prevent future access to it. | |
92 | let instance = fx.instance; | |
93 | let mut clif_comments = fx.clif_comments; | |
94 | let source_info_set = fx.source_info_set; | |
95 | let local_map = fx.local_map; | |
96 | let cold_blocks = fx.cold_blocks; | |
97 | ||
98 | // Store function in context | |
99 | let context = &mut cx.cached_context; | |
100 | context.func = func; | |
101 | ||
102 | crate::pretty_clif::write_clif_file(tcx, "unopt", None, instance, &context, &clif_comments); | |
103 | ||
104 | // Verify function | |
105 | verify_func(tcx, &clif_comments, &context.func); | |
106 | ||
107 | // Perform rust specific optimizations | |
108 | tcx.sess.time("optimize clif ir", || { | |
109 | crate::optimize::optimize_function( | |
110 | tcx, | |
111 | instance, | |
112 | context, | |
113 | &cold_blocks, | |
114 | &mut clif_comments, | |
115 | ); | |
116 | }); | |
117 | ||
118 | // If the return block is not reachable, then the SSA builder may have inserted an `iconst.i128` | |
119 | // instruction, which doesn't have an encoding. | |
120 | context.compute_cfg(); | |
121 | context.compute_domtree(); | |
122 | context.eliminate_unreachable_code(cx.module.isa()).unwrap(); | |
123 | context.dce(cx.module.isa()).unwrap(); | |
5869c6ff XL |
124 | // Some Cranelift optimizations expect the domtree to not yet be computed and as such don't |
125 | // invalidate it when it would change. | |
126 | context.domtree.clear(); | |
127 | ||
128 | context.want_disasm = crate::pretty_clif::should_write_ir(tcx); | |
29967ef6 XL |
129 | |
130 | // Define function | |
131 | let module = &mut cx.module; | |
132 | tcx.sess.time("define function", || { | |
133 | module | |
6a06907d | 134 | .define_function(func_id, context, &mut cranelift_codegen::binemit::NullTrapSink {}) |
29967ef6 XL |
135 | .unwrap() |
136 | }); | |
137 | ||
138 | // Write optimized function to file for debugging | |
139 | crate::pretty_clif::write_clif_file( | |
140 | tcx, | |
141 | "opt", | |
142 | Some(cx.module.isa()), | |
143 | instance, | |
144 | &context, | |
145 | &clif_comments, | |
146 | ); | |
147 | ||
6a06907d XL |
148 | if let Some(disasm) = &context.mach_compile_result.as_ref().unwrap().disasm { |
149 | crate::pretty_clif::write_ir_file( | |
150 | tcx, | |
151 | &format!("{}.vcode", tcx.symbol_name(instance).name), | |
152 | |file| file.write_all(disasm.as_bytes()), | |
153 | ) | |
5869c6ff XL |
154 | } |
155 | ||
29967ef6 XL |
156 | // Define debuginfo for function |
157 | let isa = cx.module.isa(); | |
158 | let debug_context = &mut cx.debug_context; | |
159 | let unwind_context = &mut cx.unwind_context; | |
160 | tcx.sess.time("generate debug info", || { | |
161 | if let Some(debug_context) = debug_context { | |
162 | debug_context.define_function( | |
163 | instance, | |
164 | func_id, | |
165 | &name, | |
166 | isa, | |
167 | context, | |
168 | &source_info_set, | |
169 | local_map, | |
170 | ); | |
171 | } | |
172 | unwind_context.add_function(func_id, &context, isa); | |
173 | }); | |
174 | ||
175 | // Clear context to make it usable for the next function | |
176 | context.clear(); | |
177 | } | |
178 | ||
179 | pub(crate) fn verify_func( | |
180 | tcx: TyCtxt<'_>, | |
181 | writer: &crate::pretty_clif::CommentWriter, | |
182 | func: &Function, | |
183 | ) { | |
184 | tcx.sess.time("verify clif ir", || { | |
185 | let flags = cranelift_codegen::settings::Flags::new(cranelift_codegen::settings::builder()); | |
186 | match cranelift_codegen::verify_function(&func, &flags) { | |
187 | Ok(_) => {} | |
188 | Err(err) => { | |
189 | tcx.sess.err(&format!("{:?}", err)); | |
190 | let pretty_error = cranelift_codegen::print_errors::pretty_verifier_error( | |
191 | &func, | |
192 | None, | |
193 | Some(Box::new(writer)), | |
194 | err, | |
195 | ); | |
6a06907d | 196 | tcx.sess.fatal(&format!("cranelift verify error:\n{}", pretty_error)); |
29967ef6 XL |
197 | } |
198 | } | |
199 | }); | |
200 | } | |
201 | ||
6a06907d | 202 | fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) { |
29967ef6 XL |
203 | for (bb, bb_data) in fx.mir.basic_blocks().iter_enumerated() { |
204 | let block = fx.get_block(bb); | |
205 | fx.bcx.switch_to_block(block); | |
206 | ||
207 | if bb_data.is_cleanup { | |
208 | // Unwinding after panicking is not supported | |
209 | continue; | |
210 | ||
211 | // FIXME once unwinding is supported uncomment next lines | |
212 | // // Unwinding is unlikely to happen, so mark cleanup block's as cold. | |
213 | // fx.cold_blocks.insert(block); | |
214 | } | |
215 | ||
216 | fx.bcx.ins().nop(); | |
217 | for stmt in &bb_data.statements { | |
218 | fx.set_debug_loc(stmt.source_info); | |
219 | codegen_stmt(fx, block, stmt); | |
220 | } | |
221 | ||
222 | #[cfg(debug_assertions)] | |
223 | { | |
224 | let mut terminator_head = "\n".to_string(); | |
6a06907d | 225 | bb_data.terminator().kind.fmt_head(&mut terminator_head).unwrap(); |
29967ef6 XL |
226 | let inst = fx.bcx.func.layout.last_inst(block).unwrap(); |
227 | fx.add_comment(inst, terminator_head); | |
228 | } | |
229 | ||
230 | fx.set_debug_loc(bb_data.terminator().source_info); | |
231 | ||
232 | match &bb_data.terminator().kind { | |
233 | TerminatorKind::Goto { target } => { | |
234 | if let TerminatorKind::Return = fx.mir[*target].terminator().kind { | |
235 | let mut can_immediately_return = true; | |
236 | for stmt in &fx.mir[*target].statements { | |
237 | if let StatementKind::StorageDead(_) = stmt.kind { | |
238 | } else { | |
239 | // FIXME Can sometimes happen, see rust-lang/rust#70531 | |
240 | can_immediately_return = false; | |
241 | break; | |
242 | } | |
243 | } | |
244 | ||
245 | if can_immediately_return { | |
246 | crate::abi::codegen_return(fx); | |
247 | continue; | |
248 | } | |
249 | } | |
250 | ||
251 | let block = fx.get_block(*target); | |
252 | fx.bcx.ins().jump(block, &[]); | |
253 | } | |
254 | TerminatorKind::Return => { | |
255 | crate::abi::codegen_return(fx); | |
256 | } | |
6a06907d | 257 | TerminatorKind::Assert { cond, expected, msg, target, cleanup: _ } => { |
29967ef6 XL |
258 | if !fx.tcx.sess.overflow_checks() { |
259 | if let mir::AssertKind::OverflowNeg(_) = *msg { | |
260 | let target = fx.get_block(*target); | |
261 | fx.bcx.ins().jump(target, &[]); | |
262 | continue; | |
263 | } | |
264 | } | |
265 | let cond = codegen_operand(fx, cond).load_scalar(fx); | |
266 | ||
267 | let target = fx.get_block(*target); | |
268 | let failure = fx.bcx.create_block(); | |
269 | fx.cold_blocks.insert(failure); | |
270 | ||
271 | if *expected { | |
272 | fx.bcx.ins().brz(cond, failure, &[]); | |
273 | } else { | |
274 | fx.bcx.ins().brnz(cond, failure, &[]); | |
275 | }; | |
276 | fx.bcx.ins().jump(target, &[]); | |
277 | ||
278 | fx.bcx.switch_to_block(failure); | |
279 | fx.bcx.ins().nop(); | |
280 | ||
281 | match msg { | |
282 | AssertKind::BoundsCheck { ref len, ref index } => { | |
283 | let len = codegen_operand(fx, len).load_scalar(fx); | |
284 | let index = codegen_operand(fx, index).load_scalar(fx); | |
285 | let location = fx | |
286 | .get_caller_location(bb_data.terminator().source_info.span) | |
287 | .load_scalar(fx); | |
288 | ||
289 | codegen_panic_inner( | |
290 | fx, | |
291 | rustc_hir::LangItem::PanicBoundsCheck, | |
292 | &[index, len, location], | |
293 | bb_data.terminator().source_info.span, | |
294 | ); | |
295 | } | |
296 | _ => { | |
297 | let msg_str = msg.description(); | |
298 | codegen_panic(fx, msg_str, bb_data.terminator().source_info.span); | |
299 | } | |
300 | } | |
301 | } | |
302 | ||
6a06907d | 303 | TerminatorKind::SwitchInt { discr, switch_ty, targets } => { |
29967ef6 XL |
304 | let discr = codegen_operand(fx, discr).load_scalar(fx); |
305 | ||
5869c6ff XL |
306 | let use_bool_opt = switch_ty.kind() == fx.tcx.types.bool.kind() |
307 | || (targets.iter().count() == 1 && targets.iter().next().unwrap().0 == 0); | |
308 | if use_bool_opt { | |
29967ef6 XL |
309 | assert_eq!(targets.iter().count(), 1); |
310 | let (then_value, then_block) = targets.iter().next().unwrap(); | |
311 | let then_block = fx.get_block(then_block); | |
312 | let else_block = fx.get_block(targets.otherwise()); | |
313 | let test_zero = match then_value { | |
314 | 0 => true, | |
315 | 1 => false, | |
316 | _ => unreachable!("{:?}", targets), | |
317 | }; | |
318 | ||
319 | let discr = crate::optimize::peephole::maybe_unwrap_bint(&mut fx.bcx, discr); | |
320 | let (discr, is_inverted) = | |
321 | crate::optimize::peephole::maybe_unwrap_bool_not(&mut fx.bcx, discr); | |
322 | let test_zero = if is_inverted { !test_zero } else { test_zero }; | |
323 | let discr = crate::optimize::peephole::maybe_unwrap_bint(&mut fx.bcx, discr); | |
324 | let discr = | |
325 | crate::optimize::peephole::make_branchable_value(&mut fx.bcx, discr); | |
5869c6ff XL |
326 | if let Some(taken) = crate::optimize::peephole::maybe_known_branch_taken( |
327 | &fx.bcx, discr, test_zero, | |
328 | ) { | |
329 | if taken { | |
330 | fx.bcx.ins().jump(then_block, &[]); | |
331 | } else { | |
332 | fx.bcx.ins().jump(else_block, &[]); | |
333 | } | |
29967ef6 | 334 | } else { |
5869c6ff XL |
335 | if test_zero { |
336 | fx.bcx.ins().brz(discr, then_block, &[]); | |
337 | fx.bcx.ins().jump(else_block, &[]); | |
338 | } else { | |
339 | fx.bcx.ins().brnz(discr, then_block, &[]); | |
340 | fx.bcx.ins().jump(else_block, &[]); | |
341 | } | |
29967ef6 XL |
342 | } |
343 | } else { | |
344 | let mut switch = ::cranelift_frontend::Switch::new(); | |
345 | for (value, block) in targets.iter() { | |
346 | let block = fx.get_block(block); | |
347 | switch.set_entry(value, block); | |
348 | } | |
349 | let otherwise_block = fx.get_block(targets.otherwise()); | |
350 | switch.emit(&mut fx.bcx, discr, otherwise_block); | |
351 | } | |
352 | } | |
353 | TerminatorKind::Call { | |
354 | func, | |
355 | args, | |
356 | destination, | |
357 | fn_span, | |
358 | cleanup: _, | |
359 | from_hir_call: _, | |
360 | } => { | |
361 | fx.tcx.sess.time("codegen call", || { | |
362 | crate::abi::codegen_terminator_call( | |
363 | fx, | |
364 | *fn_span, | |
365 | block, | |
366 | func, | |
367 | args, | |
368 | *destination, | |
369 | ) | |
370 | }); | |
371 | } | |
372 | TerminatorKind::InlineAsm { | |
373 | template, | |
374 | operands, | |
375 | options, | |
376 | destination, | |
377 | line_spans: _, | |
378 | } => { | |
379 | crate::inline_asm::codegen_inline_asm( | |
380 | fx, | |
381 | bb_data.terminator().source_info.span, | |
382 | template, | |
383 | operands, | |
384 | *options, | |
385 | ); | |
386 | ||
387 | match *destination { | |
388 | Some(destination) => { | |
389 | let destination_block = fx.get_block(destination); | |
390 | fx.bcx.ins().jump(destination_block, &[]); | |
391 | } | |
392 | None => { | |
393 | crate::trap::trap_unreachable( | |
394 | fx, | |
395 | "[corruption] Returned from noreturn inline asm", | |
396 | ); | |
397 | } | |
398 | } | |
399 | } | |
400 | TerminatorKind::Resume | TerminatorKind::Abort => { | |
401 | trap_unreachable(fx, "[corruption] Unwinding bb reached."); | |
402 | } | |
403 | TerminatorKind::Unreachable => { | |
404 | trap_unreachable(fx, "[corruption] Hit unreachable code."); | |
405 | } | |
406 | TerminatorKind::Yield { .. } | |
407 | | TerminatorKind::FalseEdge { .. } | |
408 | | TerminatorKind::FalseUnwind { .. } | |
409 | | TerminatorKind::DropAndReplace { .. } | |
410 | | TerminatorKind::GeneratorDrop => { | |
411 | bug!("shouldn't exist at codegen {:?}", bb_data.terminator()); | |
412 | } | |
6a06907d | 413 | TerminatorKind::Drop { place, target, unwind: _ } => { |
29967ef6 XL |
414 | let drop_place = codegen_place(fx, *place); |
415 | crate::abi::codegen_drop(fx, bb_data.terminator().source_info.span, drop_place); | |
416 | ||
417 | let target_block = fx.get_block(*target); | |
418 | fx.bcx.ins().jump(target_block, &[]); | |
419 | } | |
420 | }; | |
421 | } | |
422 | ||
423 | fx.bcx.seal_all_blocks(); | |
424 | fx.bcx.finalize(); | |
425 | } | |
426 | ||
427 | fn codegen_stmt<'tcx>( | |
6a06907d | 428 | fx: &mut FunctionCx<'_, '_, 'tcx>, |
29967ef6 XL |
429 | #[allow(unused_variables)] cur_block: Block, |
430 | stmt: &Statement<'tcx>, | |
431 | ) { | |
432 | let _print_guard = crate::PrintOnPanic(|| format!("stmt {:?}", stmt)); | |
433 | ||
434 | fx.set_debug_loc(stmt.source_info); | |
435 | ||
436 | #[cfg(false_debug_assertions)] | |
437 | match &stmt.kind { | |
438 | StatementKind::StorageLive(..) | StatementKind::StorageDead(..) => {} // Those are not very useful | |
439 | _ => { | |
440 | let inst = fx.bcx.func.layout.last_inst(cur_block).unwrap(); | |
441 | fx.add_comment(inst, format!("{:?}", stmt)); | |
442 | } | |
443 | } | |
444 | ||
445 | match &stmt.kind { | |
6a06907d | 446 | StatementKind::SetDiscriminant { place, variant_index } => { |
29967ef6 XL |
447 | let place = codegen_place(fx, **place); |
448 | crate::discriminant::codegen_set_discriminant(fx, place, *variant_index); | |
449 | } | |
450 | StatementKind::Assign(to_place_and_rval) => { | |
451 | let lval = codegen_place(fx, to_place_and_rval.0); | |
452 | let dest_layout = lval.layout(); | |
fc512014 XL |
453 | match to_place_and_rval.1 { |
454 | Rvalue::Use(ref operand) => { | |
29967ef6 XL |
455 | let val = codegen_operand(fx, operand); |
456 | lval.write_cvalue(fx, val); | |
457 | } | |
458 | Rvalue::Ref(_, _, place) | Rvalue::AddressOf(_, place) => { | |
fc512014 | 459 | let place = codegen_place(fx, place); |
29967ef6 XL |
460 | let ref_ = place.place_ref(fx, lval.layout()); |
461 | lval.write_cvalue(fx, ref_); | |
462 | } | |
463 | Rvalue::ThreadLocalRef(def_id) => { | |
fc512014 | 464 | let val = crate::constant::codegen_tls_ref(fx, def_id, lval.layout()); |
29967ef6 XL |
465 | lval.write_cvalue(fx, val); |
466 | } | |
6a06907d | 467 | Rvalue::BinaryOp(bin_op, box (ref lhs, ref rhs)) => { |
29967ef6 XL |
468 | let lhs = codegen_operand(fx, lhs); |
469 | let rhs = codegen_operand(fx, rhs); | |
470 | ||
fc512014 | 471 | let res = crate::num::codegen_binop(fx, bin_op, lhs, rhs); |
29967ef6 XL |
472 | lval.write_cvalue(fx, res); |
473 | } | |
6a06907d | 474 | Rvalue::CheckedBinaryOp(bin_op, box (ref lhs, ref rhs)) => { |
29967ef6 XL |
475 | let lhs = codegen_operand(fx, lhs); |
476 | let rhs = codegen_operand(fx, rhs); | |
477 | ||
478 | let res = if !fx.tcx.sess.overflow_checks() { | |
479 | let val = | |
fc512014 | 480 | crate::num::codegen_int_binop(fx, bin_op, lhs, rhs).load_scalar(fx); |
29967ef6 XL |
481 | let is_overflow = fx.bcx.ins().iconst(types::I8, 0); |
482 | CValue::by_val_pair(val, is_overflow, lval.layout()) | |
483 | } else { | |
fc512014 | 484 | crate::num::codegen_checked_int_binop(fx, bin_op, lhs, rhs) |
29967ef6 XL |
485 | }; |
486 | ||
487 | lval.write_cvalue(fx, res); | |
488 | } | |
fc512014 | 489 | Rvalue::UnaryOp(un_op, ref operand) => { |
29967ef6 XL |
490 | let operand = codegen_operand(fx, operand); |
491 | let layout = operand.layout(); | |
492 | let val = operand.load_scalar(fx); | |
493 | let res = match un_op { | |
494 | UnOp::Not => match layout.ty.kind() { | |
495 | ty::Bool => { | |
496 | let res = fx.bcx.ins().icmp_imm(IntCC::Equal, val, 0); | |
497 | CValue::by_val(fx.bcx.ins().bint(types::I8, res), layout) | |
498 | } | |
499 | ty::Uint(_) | ty::Int(_) => { | |
500 | CValue::by_val(fx.bcx.ins().bnot(val), layout) | |
501 | } | |
502 | _ => unreachable!("un op Not for {:?}", layout.ty), | |
503 | }, | |
504 | UnOp::Neg => match layout.ty.kind() { | |
505 | ty::Int(IntTy::I128) => { | |
506 | // FIXME remove this case once ineg.i128 works | |
fc512014 XL |
507 | let zero = |
508 | CValue::const_val(fx, layout, ty::ScalarInt::null(layout.size)); | |
29967ef6 XL |
509 | crate::num::codegen_int_binop(fx, BinOp::Sub, zero, operand) |
510 | } | |
511 | ty::Int(_) => CValue::by_val(fx.bcx.ins().ineg(val), layout), | |
512 | ty::Float(_) => CValue::by_val(fx.bcx.ins().fneg(val), layout), | |
513 | _ => unreachable!("un op Neg for {:?}", layout.ty), | |
514 | }, | |
515 | }; | |
516 | lval.write_cvalue(fx, res); | |
517 | } | |
fc512014 XL |
518 | Rvalue::Cast( |
519 | CastKind::Pointer(PointerCast::ReifyFnPointer), | |
520 | ref operand, | |
521 | to_ty, | |
522 | ) => { | |
523 | let from_ty = fx.monomorphize(operand.ty(&fx.mir.local_decls, fx.tcx)); | |
29967ef6 XL |
524 | let to_layout = fx.layout_of(fx.monomorphize(to_ty)); |
525 | match *from_ty.kind() { | |
526 | ty::FnDef(def_id, substs) => { | |
527 | let func_ref = fx.get_function_ref( | |
528 | Instance::resolve_for_fn_ptr( | |
529 | fx.tcx, | |
530 | ParamEnv::reveal_all(), | |
531 | def_id, | |
532 | substs, | |
533 | ) | |
534 | .unwrap() | |
535 | .polymorphize(fx.tcx), | |
536 | ); | |
537 | let func_addr = fx.bcx.ins().func_addr(fx.pointer_type, func_ref); | |
538 | lval.write_cvalue(fx, CValue::by_val(func_addr, to_layout)); | |
539 | } | |
540 | _ => bug!("Trying to ReifyFnPointer on non FnDef {:?}", from_ty), | |
541 | } | |
542 | } | |
fc512014 XL |
543 | Rvalue::Cast( |
544 | CastKind::Pointer(PointerCast::UnsafeFnPointer), | |
545 | ref operand, | |
546 | to_ty, | |
547 | ) | |
548 | | Rvalue::Cast( | |
549 | CastKind::Pointer(PointerCast::MutToConstPointer), | |
550 | ref operand, | |
551 | to_ty, | |
552 | ) | |
553 | | Rvalue::Cast( | |
554 | CastKind::Pointer(PointerCast::ArrayToPointer), | |
555 | ref operand, | |
556 | to_ty, | |
557 | ) => { | |
29967ef6 XL |
558 | let to_layout = fx.layout_of(fx.monomorphize(to_ty)); |
559 | let operand = codegen_operand(fx, operand); | |
560 | lval.write_cvalue(fx, operand.cast_pointer_to(to_layout)); | |
561 | } | |
fc512014 | 562 | Rvalue::Cast(CastKind::Misc, ref operand, to_ty) => { |
29967ef6 XL |
563 | let operand = codegen_operand(fx, operand); |
564 | let from_ty = operand.layout().ty; | |
565 | let to_ty = fx.monomorphize(to_ty); | |
566 | ||
6a06907d | 567 | fn is_fat_ptr<'tcx>(fx: &FunctionCx<'_, '_, 'tcx>, ty: Ty<'tcx>) -> bool { |
29967ef6 | 568 | ty.builtin_deref(true) |
6a06907d XL |
569 | .map(|ty::TypeAndMut { ty: pointee_ty, mutbl: _ }| { |
570 | has_ptr_meta(fx.tcx, pointee_ty) | |
571 | }) | |
29967ef6 XL |
572 | .unwrap_or(false) |
573 | } | |
574 | ||
575 | if is_fat_ptr(fx, from_ty) { | |
576 | if is_fat_ptr(fx, to_ty) { | |
577 | // fat-ptr -> fat-ptr | |
578 | lval.write_cvalue(fx, operand.cast_pointer_to(dest_layout)); | |
579 | } else { | |
580 | // fat-ptr -> thin-ptr | |
581 | let (ptr, _extra) = operand.load_scalar_pair(fx); | |
582 | lval.write_cvalue(fx, CValue::by_val(ptr, dest_layout)) | |
583 | } | |
584 | } else if let ty::Adt(adt_def, _substs) = from_ty.kind() { | |
585 | // enum -> discriminant value | |
586 | assert!(adt_def.is_enum()); | |
587 | match to_ty.kind() { | |
588 | ty::Uint(_) | ty::Int(_) => {} | |
589 | _ => unreachable!("cast adt {} -> {}", from_ty, to_ty), | |
590 | } | |
6a06907d | 591 | let to_clif_ty = fx.clif_type(to_ty).unwrap(); |
29967ef6 | 592 | |
6a06907d XL |
593 | let discriminant = crate::discriminant::codegen_get_discriminant( |
594 | fx, | |
595 | operand, | |
596 | fx.layout_of(operand.layout().ty.discriminant_ty(fx.tcx)), | |
597 | ) | |
598 | .load_scalar(fx); | |
599 | ||
600 | let res = crate::cast::clif_intcast( | |
601 | fx, | |
602 | discriminant, | |
603 | to_clif_ty, | |
604 | to_ty.is_signed(), | |
605 | ); | |
606 | lval.write_cvalue(fx, CValue::by_val(res, dest_layout)); | |
29967ef6 XL |
607 | } else { |
608 | let to_clif_ty = fx.clif_type(to_ty).unwrap(); | |
609 | let from = operand.load_scalar(fx); | |
610 | ||
611 | let res = clif_int_or_float_cast( | |
612 | fx, | |
613 | from, | |
614 | type_sign(from_ty), | |
615 | to_clif_ty, | |
616 | type_sign(to_ty), | |
617 | ); | |
618 | lval.write_cvalue(fx, CValue::by_val(res, dest_layout)); | |
619 | } | |
620 | } | |
621 | Rvalue::Cast( | |
622 | CastKind::Pointer(PointerCast::ClosureFnPointer(_)), | |
fc512014 | 623 | ref operand, |
29967ef6 XL |
624 | _to_ty, |
625 | ) => { | |
626 | let operand = codegen_operand(fx, operand); | |
627 | match *operand.layout().ty.kind() { | |
628 | ty::Closure(def_id, substs) => { | |
629 | let instance = Instance::resolve_closure( | |
630 | fx.tcx, | |
631 | def_id, | |
632 | substs, | |
633 | ty::ClosureKind::FnOnce, | |
634 | ) | |
635 | .polymorphize(fx.tcx); | |
636 | let func_ref = fx.get_function_ref(instance); | |
637 | let func_addr = fx.bcx.ins().func_addr(fx.pointer_type, func_ref); | |
638 | lval.write_cvalue(fx, CValue::by_val(func_addr, lval.layout())); | |
639 | } | |
640 | _ => bug!("{} cannot be cast to a fn ptr", operand.layout().ty), | |
641 | } | |
642 | } | |
fc512014 | 643 | Rvalue::Cast(CastKind::Pointer(PointerCast::Unsize), ref operand, _to_ty) => { |
29967ef6 XL |
644 | let operand = codegen_operand(fx, operand); |
645 | operand.unsize_value(fx, lval); | |
646 | } | |
647 | Rvalue::Discriminant(place) => { | |
fc512014 | 648 | let place = codegen_place(fx, place); |
29967ef6 XL |
649 | let value = place.to_cvalue(fx); |
650 | let discr = | |
651 | crate::discriminant::codegen_get_discriminant(fx, value, dest_layout); | |
652 | lval.write_cvalue(fx, discr); | |
653 | } | |
fc512014 | 654 | Rvalue::Repeat(ref operand, times) => { |
29967ef6 XL |
655 | let operand = codegen_operand(fx, operand); |
656 | let times = fx | |
657 | .monomorphize(times) | |
658 | .eval(fx.tcx, ParamEnv::reveal_all()) | |
659 | .val | |
660 | .try_to_bits(fx.tcx.data_layout.pointer_size) | |
661 | .unwrap(); | |
662 | if fx.clif_type(operand.layout().ty) == Some(types::I8) { | |
663 | let times = fx.bcx.ins().iconst(fx.pointer_type, times as i64); | |
664 | // FIXME use emit_small_memset where possible | |
665 | let addr = lval.to_ptr().get_addr(fx); | |
666 | let val = operand.load_scalar(fx); | |
6a06907d | 667 | fx.bcx.call_memset(fx.cx.module.target_config(), addr, val, times); |
29967ef6 XL |
668 | } else { |
669 | let loop_block = fx.bcx.create_block(); | |
670 | let loop_block2 = fx.bcx.create_block(); | |
671 | let done_block = fx.bcx.create_block(); | |
672 | let index = fx.bcx.append_block_param(loop_block, fx.pointer_type); | |
673 | let zero = fx.bcx.ins().iconst(fx.pointer_type, 0); | |
674 | fx.bcx.ins().jump(loop_block, &[zero]); | |
675 | ||
676 | fx.bcx.switch_to_block(loop_block); | |
677 | let done = fx.bcx.ins().icmp_imm(IntCC::Equal, index, times as i64); | |
678 | fx.bcx.ins().brnz(done, done_block, &[]); | |
679 | fx.bcx.ins().jump(loop_block2, &[]); | |
680 | ||
681 | fx.bcx.switch_to_block(loop_block2); | |
682 | let to = lval.place_index(fx, index); | |
683 | to.write_cvalue(fx, operand); | |
684 | let index = fx.bcx.ins().iadd_imm(index, 1); | |
685 | fx.bcx.ins().jump(loop_block, &[index]); | |
686 | ||
687 | fx.bcx.switch_to_block(done_block); | |
688 | fx.bcx.ins().nop(); | |
689 | } | |
690 | } | |
691 | Rvalue::Len(place) => { | |
fc512014 | 692 | let place = codegen_place(fx, place); |
29967ef6 XL |
693 | let usize_layout = fx.layout_of(fx.tcx.types.usize); |
694 | let len = codegen_array_len(fx, place); | |
695 | lval.write_cvalue(fx, CValue::by_val(len, usize_layout)); | |
696 | } | |
697 | Rvalue::NullaryOp(NullOp::Box, content_ty) => { | |
698 | let usize_type = fx.clif_type(fx.tcx.types.usize).unwrap(); | |
699 | let content_ty = fx.monomorphize(content_ty); | |
700 | let layout = fx.layout_of(content_ty); | |
701 | let llsize = fx.bcx.ins().iconst(usize_type, layout.size.bytes() as i64); | |
6a06907d | 702 | let llalign = fx.bcx.ins().iconst(usize_type, layout.align.abi.bytes() as i64); |
29967ef6 XL |
703 | let box_layout = fx.layout_of(fx.tcx.mk_box(content_ty)); |
704 | ||
705 | // Allocate space: | |
6a06907d XL |
706 | let def_id = |
707 | match fx.tcx.lang_items().require(rustc_hir::LangItem::ExchangeMalloc) { | |
708 | Ok(id) => id, | |
709 | Err(s) => { | |
710 | fx.tcx | |
711 | .sess | |
712 | .fatal(&format!("allocation of `{}` {}", box_layout.ty, s)); | |
713 | } | |
714 | }; | |
29967ef6 XL |
715 | let instance = ty::Instance::mono(fx.tcx, def_id).polymorphize(fx.tcx); |
716 | let func_ref = fx.get_function_ref(instance); | |
717 | let call = fx.bcx.ins().call(func_ref, &[llsize, llalign]); | |
718 | let ptr = fx.bcx.inst_results(call)[0]; | |
719 | lval.write_cvalue(fx, CValue::by_val(ptr, box_layout)); | |
720 | } | |
721 | Rvalue::NullaryOp(NullOp::SizeOf, ty) => { | |
6a06907d XL |
722 | assert!( |
723 | lval.layout() | |
724 | .ty | |
725 | .is_sized(fx.tcx.at(stmt.source_info.span), ParamEnv::reveal_all()) | |
726 | ); | |
29967ef6 XL |
727 | let ty_size = fx.layout_of(fx.monomorphize(ty)).size.bytes(); |
728 | let val = | |
729 | CValue::const_val(fx, fx.layout_of(fx.tcx.types.usize), ty_size.into()); | |
730 | lval.write_cvalue(fx, val); | |
731 | } | |
fc512014 | 732 | Rvalue::Aggregate(ref kind, ref operands) => match kind.as_ref() { |
29967ef6 XL |
733 | AggregateKind::Array(_ty) => { |
734 | for (i, operand) in operands.iter().enumerate() { | |
735 | let operand = codegen_operand(fx, operand); | |
736 | let index = fx.bcx.ins().iconst(fx.pointer_type, i as i64); | |
737 | let to = lval.place_index(fx, index); | |
738 | to.write_cvalue(fx, operand); | |
739 | } | |
740 | } | |
741 | _ => unreachable!("shouldn't exist at codegen {:?}", to_place_and_rval.1), | |
742 | }, | |
743 | } | |
744 | } | |
745 | StatementKind::StorageLive(_) | |
746 | | StatementKind::StorageDead(_) | |
747 | | StatementKind::Nop | |
748 | | StatementKind::FakeRead(..) | |
749 | | StatementKind::Retag { .. } | |
750 | | StatementKind::AscribeUserType(..) => {} | |
751 | ||
752 | StatementKind::LlvmInlineAsm(asm) => { | |
753 | use rustc_span::symbol::Symbol; | |
6a06907d | 754 | let LlvmInlineAsm { asm, outputs, inputs } = &**asm; |
29967ef6 XL |
755 | let rustc_hir::LlvmInlineAsmInner { |
756 | asm: asm_code, // Name | |
757 | outputs: output_names, // Vec<LlvmInlineAsmOutput> | |
758 | inputs: input_names, // Vec<Name> | |
759 | clobbers, // Vec<Name> | |
760 | volatile, // bool | |
761 | alignstack, // bool | |
762 | dialect: _, | |
763 | asm_str_style: _, | |
764 | } = asm; | |
765 | match asm_code.as_str().trim() { | |
766 | "" => { | |
767 | // Black box | |
768 | } | |
769 | "mov %rbx, %rsi\n cpuid\n xchg %rbx, %rsi" => { | |
6a06907d | 770 | assert_eq!(input_names, &[Symbol::intern("{eax}"), Symbol::intern("{ecx}")]); |
29967ef6 | 771 | assert_eq!(output_names.len(), 4); |
6a06907d | 772 | for (i, c) in (&["={eax}", "={esi}", "={ecx}", "={edx}"]).iter().enumerate() { |
29967ef6 XL |
773 | assert_eq!(&output_names[i].constraint.as_str(), c); |
774 | assert!(!output_names[i].is_rw); | |
775 | assert!(!output_names[i].is_indirect); | |
776 | } | |
777 | ||
778 | assert_eq!(clobbers, &[]); | |
779 | ||
780 | assert!(!volatile); | |
781 | assert!(!alignstack); | |
782 | ||
783 | assert_eq!(inputs.len(), 2); | |
784 | let leaf = codegen_operand(fx, &inputs[0].1).load_scalar(fx); // %eax | |
785 | let subleaf = codegen_operand(fx, &inputs[1].1).load_scalar(fx); // %ecx | |
786 | ||
787 | let (eax, ebx, ecx, edx) = | |
788 | crate::intrinsics::codegen_cpuid_call(fx, leaf, subleaf); | |
789 | ||
790 | assert_eq!(outputs.len(), 4); | |
791 | codegen_place(fx, outputs[0]) | |
792 | .write_cvalue(fx, CValue::by_val(eax, fx.layout_of(fx.tcx.types.u32))); | |
793 | codegen_place(fx, outputs[1]) | |
794 | .write_cvalue(fx, CValue::by_val(ebx, fx.layout_of(fx.tcx.types.u32))); | |
795 | codegen_place(fx, outputs[2]) | |
796 | .write_cvalue(fx, CValue::by_val(ecx, fx.layout_of(fx.tcx.types.u32))); | |
797 | codegen_place(fx, outputs[3]) | |
798 | .write_cvalue(fx, CValue::by_val(edx, fx.layout_of(fx.tcx.types.u32))); | |
799 | } | |
800 | "xgetbv" => { | |
801 | assert_eq!(input_names, &[Symbol::intern("{ecx}")]); | |
802 | ||
803 | assert_eq!(output_names.len(), 2); | |
804 | for (i, c) in (&["={eax}", "={edx}"]).iter().enumerate() { | |
805 | assert_eq!(&output_names[i].constraint.as_str(), c); | |
806 | assert!(!output_names[i].is_rw); | |
807 | assert!(!output_names[i].is_indirect); | |
808 | } | |
809 | ||
810 | assert_eq!(clobbers, &[]); | |
811 | ||
812 | assert!(!volatile); | |
813 | assert!(!alignstack); | |
814 | ||
815 | crate::trap::trap_unimplemented(fx, "_xgetbv arch intrinsic is not supported"); | |
816 | } | |
817 | // ___chkstk, ___chkstk_ms and __alloca are only used on Windows | |
6a06907d | 818 | _ if fx.tcx.symbol_name(fx.instance).name.starts_with("___chkstk") => { |
29967ef6 XL |
819 | crate::trap::trap_unimplemented(fx, "Stack probes are not supported"); |
820 | } | |
821 | _ if fx.tcx.symbol_name(fx.instance).name == "__alloca" => { | |
822 | crate::trap::trap_unimplemented(fx, "Alloca is not supported"); | |
823 | } | |
824 | // Used in sys::windows::abort_internal | |
825 | "int $$0x29" => { | |
826 | crate::trap::trap_unimplemented(fx, "Windows abort"); | |
827 | } | |
828 | _ => fx | |
829 | .tcx | |
830 | .sess | |
831 | .span_fatal(stmt.source_info.span, "Inline assembly is not supported"), | |
832 | } | |
833 | } | |
834 | StatementKind::Coverage { .. } => fx.tcx.sess.fatal("-Zcoverage is unimplemented"), | |
6a06907d XL |
835 | StatementKind::CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping { |
836 | src, | |
837 | dst, | |
838 | count, | |
839 | }) => { | |
840 | let dst = codegen_operand(fx, dst); | |
841 | let pointee = dst | |
842 | .layout() | |
843 | .pointee_info_at(fx, rustc_target::abi::Size::ZERO) | |
844 | .expect("Expected pointer"); | |
845 | let dst = dst.load_scalar(fx); | |
846 | let src = codegen_operand(fx, src).load_scalar(fx); | |
847 | let count = codegen_operand(fx, count).load_scalar(fx); | |
848 | let elem_size: u64 = pointee.size.bytes(); | |
849 | let bytes = if elem_size != 1 { | |
850 | fx.bcx.ins().imul_imm(count, elem_size as i64) | |
851 | } else { | |
852 | count | |
853 | }; | |
854 | fx.bcx.call_memcpy(fx.cx.module.target_config(), dst, src, bytes); | |
855 | } | |
29967ef6 XL |
856 | } |
857 | } | |
858 | ||
6a06907d | 859 | fn codegen_array_len<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, place: CPlace<'tcx>) -> Value { |
29967ef6 XL |
860 | match *place.layout().ty.kind() { |
861 | ty::Array(_elem_ty, len) => { | |
6a06907d | 862 | let len = fx.monomorphize(len).eval_usize(fx.tcx, ParamEnv::reveal_all()) as i64; |
29967ef6 XL |
863 | fx.bcx.ins().iconst(fx.pointer_type, len) |
864 | } | |
6a06907d XL |
865 | ty::Slice(_elem_ty) => { |
866 | place.to_ptr_maybe_unsized().1.expect("Length metadata for slice place") | |
867 | } | |
29967ef6 XL |
868 | _ => bug!("Rvalue::Len({:?})", place), |
869 | } | |
870 | } | |
871 | ||
872 | pub(crate) fn codegen_place<'tcx>( | |
6a06907d | 873 | fx: &mut FunctionCx<'_, '_, 'tcx>, |
29967ef6 XL |
874 | place: Place<'tcx>, |
875 | ) -> CPlace<'tcx> { | |
876 | let mut cplace = fx.get_local_place(place.local); | |
877 | ||
878 | for elem in place.projection { | |
879 | match elem { | |
880 | PlaceElem::Deref => { | |
881 | cplace = cplace.place_deref(fx); | |
882 | } | |
883 | PlaceElem::Field(field, _ty) => { | |
884 | cplace = cplace.place_field(fx, field); | |
885 | } | |
886 | PlaceElem::Index(local) => { | |
887 | let index = fx.get_local_place(local).to_cvalue(fx).load_scalar(fx); | |
888 | cplace = cplace.place_index(fx, index); | |
889 | } | |
6a06907d | 890 | PlaceElem::ConstantIndex { offset, min_length: _, from_end } => { |
29967ef6 XL |
891 | let offset: u64 = offset; |
892 | let index = if !from_end { | |
893 | fx.bcx.ins().iconst(fx.pointer_type, offset as i64) | |
894 | } else { | |
895 | let len = codegen_array_len(fx, cplace); | |
896 | fx.bcx.ins().iadd_imm(len, -(offset as i64)) | |
897 | }; | |
898 | cplace = cplace.place_index(fx, index); | |
899 | } | |
900 | PlaceElem::Subslice { from, to, from_end } => { | |
901 | // These indices are generated by slice patterns. | |
902 | // slice[from:-to] in Python terms. | |
903 | ||
904 | let from: u64 = from; | |
905 | let to: u64 = to; | |
906 | ||
907 | match cplace.layout().ty.kind() { | |
908 | ty::Array(elem_ty, _len) => { | |
909 | assert!(!from_end, "array subslices are never `from_end`"); | |
910 | let elem_layout = fx.layout_of(elem_ty); | |
911 | let ptr = cplace.to_ptr(); | |
912 | cplace = CPlace::for_ptr( | |
913 | ptr.offset_i64(fx, elem_layout.size.bytes() as i64 * (from as i64)), | |
914 | fx.layout_of(fx.tcx.mk_array(elem_ty, to - from)), | |
915 | ); | |
916 | } | |
917 | ty::Slice(elem_ty) => { | |
918 | assert!(from_end, "slice subslices should be `from_end`"); | |
919 | let elem_layout = fx.layout_of(elem_ty); | |
920 | let (ptr, len) = cplace.to_ptr_maybe_unsized(); | |
921 | let len = len.unwrap(); | |
922 | cplace = CPlace::for_ptr_with_extra( | |
923 | ptr.offset_i64(fx, elem_layout.size.bytes() as i64 * (from as i64)), | |
924 | fx.bcx.ins().iadd_imm(len, -(from as i64 + to as i64)), | |
925 | cplace.layout(), | |
926 | ); | |
927 | } | |
928 | _ => unreachable!(), | |
929 | } | |
930 | } | |
931 | PlaceElem::Downcast(_adt_def, variant) => { | |
932 | cplace = cplace.downcast_variant(fx, variant); | |
933 | } | |
934 | } | |
935 | } | |
936 | ||
937 | cplace | |
938 | } | |
939 | ||
940 | pub(crate) fn codegen_operand<'tcx>( | |
6a06907d | 941 | fx: &mut FunctionCx<'_, '_, 'tcx>, |
29967ef6 XL |
942 | operand: &Operand<'tcx>, |
943 | ) -> CValue<'tcx> { | |
944 | match operand { | |
945 | Operand::Move(place) | Operand::Copy(place) => { | |
946 | let cplace = codegen_place(fx, *place); | |
947 | cplace.to_cvalue(fx) | |
948 | } | |
949 | Operand::Constant(const_) => crate::constant::codegen_constant(fx, const_), | |
950 | } | |
951 | } | |
952 | ||
6a06907d | 953 | pub(crate) fn codegen_panic<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, msg_str: &str, span: Span) { |
29967ef6 XL |
954 | let location = fx.get_caller_location(span).load_scalar(fx); |
955 | ||
956 | let msg_ptr = fx.anonymous_str("assert", msg_str); | |
6a06907d | 957 | let msg_len = fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(msg_str.len()).unwrap()); |
29967ef6 XL |
958 | let args = [msg_ptr, msg_len, location]; |
959 | ||
960 | codegen_panic_inner(fx, rustc_hir::LangItem::Panic, &args, span); | |
961 | } | |
962 | ||
963 | pub(crate) fn codegen_panic_inner<'tcx>( | |
6a06907d | 964 | fx: &mut FunctionCx<'_, '_, 'tcx>, |
29967ef6 XL |
965 | lang_item: rustc_hir::LangItem, |
966 | args: &[Value], | |
967 | span: Span, | |
968 | ) { | |
6a06907d XL |
969 | let def_id = |
970 | fx.tcx.lang_items().require(lang_item).unwrap_or_else(|s| fx.tcx.sess.span_fatal(span, &s)); | |
29967ef6 XL |
971 | |
972 | let instance = Instance::mono(fx.tcx, def_id).polymorphize(fx.tcx); | |
973 | let symbol_name = fx.tcx.symbol_name(instance).name; | |
974 | ||
975 | fx.lib_call( | |
976 | &*symbol_name, | |
5869c6ff XL |
977 | vec![ |
978 | AbiParam::new(fx.pointer_type), | |
979 | AbiParam::new(fx.pointer_type), | |
980 | AbiParam::new(fx.pointer_type), | |
981 | ], | |
29967ef6 XL |
982 | vec![], |
983 | args, | |
984 | ); | |
985 | ||
986 | crate::trap::trap_unreachable(fx, "panic lang item returned"); | |
987 | } |