]>
Commit | Line | Data |
---|---|---|
29967ef6 XL |
1 | //! Codegen of a single function |
2 | ||
a2a8927a | 3 | use rustc_ast::InlineAsmOptions; |
29967ef6 XL |
4 | use rustc_index::vec::IndexVec; |
5 | use rustc_middle::ty::adjustment::PointerCast; | |
c295e0f8 | 6 | use rustc_middle::ty::layout::FnAbiOf; |
5e7ed085 FG |
7 | use rustc_middle::ty::print::with_no_trimmed_paths; |
8 | ||
2b03887a FG |
9 | use cranelift_codegen::ir::UserFuncName; |
10 | ||
17df50a5 | 11 | use crate::constant::ConstantCx; |
f2b60f7d | 12 | use crate::debuginfo::FunctionDebugContext; |
29967ef6 | 13 | use crate::prelude::*; |
5e7ed085 | 14 | use crate::pretty_clif::CommentWriter; |
29967ef6 | 15 | |
f2b60f7d FG |
16 | pub(crate) struct CodegenedFunction { |
17 | symbol_name: String, | |
18 | func_id: FuncId, | |
19 | func: Function, | |
20 | clif_comments: CommentWriter, | |
21 | func_debug_cx: Option<FunctionDebugContext>, | |
22 | } | |
23 | ||
f2b60f7d FG |
24 | pub(crate) fn codegen_fn<'tcx>( |
25 | tcx: TyCtxt<'tcx>, | |
26 | cx: &mut crate::CodegenCx, | |
27 | cached_func: Function, | |
28 | module: &mut dyn Module, | |
29 | instance: Instance<'tcx>, | |
30 | ) -> CodegenedFunction { | |
fc512014 XL |
31 | debug_assert!(!instance.substs.needs_infer()); |
32 | ||
9ffffee4 FG |
33 | let symbol_name = tcx.symbol_name(instance).name.to_string(); |
34 | let _timer = tcx.prof.generic_activity_with_arg("codegen fn", &*symbol_name); | |
35 | ||
29967ef6 | 36 | let mir = tcx.instance_mir(instance.def); |
136023e0 XL |
37 | let _mir_guard = crate::PrintOnPanic(|| { |
38 | let mut buf = Vec::new(); | |
5e7ed085 FG |
39 | with_no_trimmed_paths!({ |
40 | rustc_middle::mir::pretty::write_mir_fn(tcx, mir, &mut |_, _| Ok(()), &mut buf) | |
41 | .unwrap(); | |
42 | }); | |
136023e0 XL |
43 | String::from_utf8_lossy(&buf).into_owned() |
44 | }); | |
29967ef6 XL |
45 | |
46 | // Declare function | |
9c376795 | 47 | let sig = get_function_sig(tcx, module.target_config().default_call_conv, instance); |
f2b60f7d | 48 | let func_id = module.declare_function(&symbol_name, Linkage::Local, &sig).unwrap(); |
29967ef6 XL |
49 | |
50 | // Make the FunctionBuilder | |
51 | let mut func_ctx = FunctionBuilderContext::new(); | |
f2b60f7d FG |
52 | let mut func = cached_func; |
53 | func.clear(); | |
2b03887a | 54 | func.name = UserFuncName::user(0, func_id.as_u32()); |
29967ef6 XL |
55 | func.signature = sig; |
56 | func.collect_debug_info(); | |
57 | ||
58 | let mut bcx = FunctionBuilder::new(&mut func, &mut func_ctx); | |
59 | ||
60 | // Predefine blocks | |
61 | let start_block = bcx.create_block(); | |
6a06907d | 62 | let block_map: IndexVec<BasicBlock, Block> = |
f2b60f7d | 63 | (0..mir.basic_blocks.len()).map(|_| bcx.create_block()).collect(); |
29967ef6 XL |
64 | |
65 | // Make FunctionCx | |
a2a8927a XL |
66 | let target_config = module.target_config(); |
67 | let pointer_type = target_config.pointer_type(); | |
29967ef6 XL |
68 | let clif_comments = crate::pretty_clif::CommentWriter::new(tcx, instance); |
69 | ||
f2b60f7d FG |
70 | let func_debug_cx = if let Some(debug_context) = &mut cx.debug_context { |
71 | Some(debug_context.define_function(tcx, &symbol_name, mir.span)) | |
72 | } else { | |
73 | None | |
74 | }; | |
75 | ||
29967ef6 XL |
76 | let mut fx = FunctionCx { |
77 | cx, | |
17df50a5 | 78 | module, |
29967ef6 | 79 | tcx, |
a2a8927a | 80 | target_config, |
29967ef6 | 81 | pointer_type, |
17df50a5 | 82 | constants_cx: ConstantCx::new(), |
f2b60f7d | 83 | func_debug_cx, |
29967ef6 XL |
84 | |
85 | instance, | |
17df50a5 | 86 | symbol_name, |
29967ef6 | 87 | mir, |
c295e0f8 | 88 | fn_abi: Some(RevealAllLayoutCx(tcx).fn_abi_of_instance(instance, ty::List::empty())), |
29967ef6 XL |
89 | |
90 | bcx, | |
91 | block_map, | |
92 | local_map: IndexVec::with_capacity(mir.local_decls.len()), | |
93 | caller_location: None, // set by `codegen_fn_prelude` | |
29967ef6 XL |
94 | |
95 | clif_comments, | |
f2b60f7d | 96 | last_source_file: None, |
29967ef6 | 97 | next_ssa_var: 0, |
29967ef6 XL |
98 | }; |
99 | ||
9ffffee4 FG |
100 | tcx.prof.generic_activity("codegen clif ir").run(|| codegen_fn_body(&mut fx, start_block)); |
101 | fx.bcx.seal_all_blocks(); | |
102 | fx.bcx.finalize(); | |
29967ef6 XL |
103 | |
104 | // Recover all necessary data from fx, before accessing func will prevent future access to it. | |
f2b60f7d | 105 | let symbol_name = fx.symbol_name; |
5e7ed085 | 106 | let clif_comments = fx.clif_comments; |
f2b60f7d | 107 | let func_debug_cx = fx.func_debug_cx; |
17df50a5 XL |
108 | |
109 | fx.constants_cx.finalize(fx.tcx, &mut *fx.module); | |
29967ef6 | 110 | |
f2b60f7d FG |
111 | if cx.should_write_ir { |
112 | crate::pretty_clif::write_clif_file( | |
113 | tcx.output_filenames(()), | |
114 | &symbol_name, | |
115 | "unopt", | |
116 | module.isa(), | |
117 | &func, | |
118 | &clif_comments, | |
119 | ); | |
120 | } | |
29967ef6 XL |
121 | |
122 | // Verify function | |
5e7ed085 FG |
123 | verify_func(tcx, &clif_comments, &func); |
124 | ||
f2b60f7d | 125 | CodegenedFunction { symbol_name, func_id, func, clif_comments, func_debug_cx } |
5e7ed085 FG |
126 | } |
127 | ||
f2b60f7d FG |
128 | pub(crate) fn compile_fn( |
129 | cx: &mut crate::CodegenCx, | |
130 | cached_context: &mut Context, | |
5e7ed085 | 131 | module: &mut dyn Module, |
f2b60f7d | 132 | codegened_func: CodegenedFunction, |
5e7ed085 | 133 | ) { |
9ffffee4 FG |
134 | let _timer = |
135 | cx.profiler.generic_activity_with_arg("compile function", &*codegened_func.symbol_name); | |
136 | ||
f2b60f7d | 137 | let clif_comments = codegened_func.clif_comments; |
5e7ed085 FG |
138 | |
139 | // Store function in context | |
f2b60f7d | 140 | let context = cached_context; |
5e7ed085 | 141 | context.clear(); |
f2b60f7d | 142 | context.func = codegened_func.func; |
29967ef6 | 143 | |
29967ef6 XL |
144 | // If the return block is not reachable, then the SSA builder may have inserted an `iconst.i128` |
145 | // instruction, which doesn't have an encoding. | |
146 | context.compute_cfg(); | |
147 | context.compute_domtree(); | |
17df50a5 XL |
148 | context.eliminate_unreachable_code(module.isa()).unwrap(); |
149 | context.dce(module.isa()).unwrap(); | |
5869c6ff XL |
150 | // Some Cranelift optimizations expect the domtree to not yet be computed and as such don't |
151 | // invalidate it when it would change. | |
152 | context.domtree.clear(); | |
153 | ||
064997fb FG |
154 | #[cfg(any())] // This is never true |
155 | let _clif_guard = { | |
156 | use std::fmt::Write; | |
157 | ||
158 | let func_clone = context.func.clone(); | |
159 | let clif_comments_clone = clif_comments.clone(); | |
160 | let mut clif = String::new(); | |
161 | for flag in module.isa().flags().iter() { | |
162 | writeln!(clif, "set {}", flag).unwrap(); | |
163 | } | |
164 | write!(clif, "target {}", module.isa().triple().architecture.to_string()).unwrap(); | |
165 | for isa_flag in module.isa().isa_flags().iter() { | |
166 | write!(clif, " {}", isa_flag).unwrap(); | |
167 | } | |
168 | writeln!(clif, "\n").unwrap(); | |
169 | crate::PrintOnPanic(move || { | |
170 | let mut clif = clif.clone(); | |
171 | ::cranelift_codegen::write::decorate_function( | |
172 | &mut &clif_comments_clone, | |
173 | &mut clif, | |
174 | &func_clone, | |
175 | ) | |
176 | .unwrap(); | |
177 | clif | |
178 | }) | |
179 | }; | |
180 | ||
29967ef6 | 181 | // Define function |
9ffffee4 | 182 | cx.profiler.generic_activity("define function").run(|| { |
f2b60f7d FG |
183 | context.want_disasm = cx.should_write_ir; |
184 | module.define_function(codegened_func.func_id, context).unwrap(); | |
9ffffee4 FG |
185 | |
186 | if cx.profiler.enabled() { | |
187 | let mut recording_args = false; | |
188 | cx.profiler | |
189 | .generic_activity_with_arg_recorder( | |
190 | "define function (clif pass timings)", | |
191 | |recorder| { | |
192 | let pass_times = cranelift_codegen::timing::take_current(); | |
193 | // Replace newlines with | as measureme doesn't allow control characters like | |
194 | // newlines inside strings. | |
353b0b11 | 195 | recorder.record_arg(format!("{}", pass_times).replace('\n', " | ")); |
9ffffee4 FG |
196 | recording_args = true; |
197 | }, | |
198 | ) | |
199 | .run(|| { | |
200 | if recording_args { | |
201 | // Wait a tiny bit to ensure chrome's profiler doesn't hide the event | |
202 | std::thread::sleep(std::time::Duration::from_nanos(2)) | |
203 | } | |
204 | }); | |
205 | } | |
29967ef6 XL |
206 | }); |
207 | ||
f2b60f7d FG |
208 | if cx.should_write_ir { |
209 | // Write optimized function to file for debugging | |
210 | crate::pretty_clif::write_clif_file( | |
211 | &cx.output_filenames, | |
212 | &codegened_func.symbol_name, | |
213 | "opt", | |
214 | module.isa(), | |
215 | &context.func, | |
216 | &clif_comments, | |
217 | ); | |
29967ef6 | 218 | |
f2b60f7d FG |
219 | if let Some(disasm) = &context.compiled_code().unwrap().disasm { |
220 | crate::pretty_clif::write_ir_file( | |
221 | &cx.output_filenames, | |
222 | &format!("{}.vcode", codegened_func.symbol_name), | |
223 | |file| file.write_all(disasm.as_bytes()), | |
224 | ) | |
225 | } | |
5869c6ff XL |
226 | } |
227 | ||
29967ef6 | 228 | // Define debuginfo for function |
17df50a5 | 229 | let isa = module.isa(); |
29967ef6 XL |
230 | let debug_context = &mut cx.debug_context; |
231 | let unwind_context = &mut cx.unwind_context; | |
9ffffee4 | 232 | cx.profiler.generic_activity("generate debug info").run(|| { |
29967ef6 | 233 | if let Some(debug_context) = debug_context { |
f2b60f7d FG |
234 | codegened_func.func_debug_cx.unwrap().finalize( |
235 | debug_context, | |
236 | codegened_func.func_id, | |
29967ef6 | 237 | context, |
29967ef6 XL |
238 | ); |
239 | } | |
f2b60f7d | 240 | unwind_context.add_function(codegened_func.func_id, &context, isa); |
29967ef6 | 241 | }); |
29967ef6 XL |
242 | } |
243 | ||
244 | pub(crate) fn verify_func( | |
245 | tcx: TyCtxt<'_>, | |
246 | writer: &crate::pretty_clif::CommentWriter, | |
247 | func: &Function, | |
248 | ) { | |
9ffffee4 | 249 | tcx.prof.generic_activity("verify clif ir").run(|| { |
29967ef6 XL |
250 | let flags = cranelift_codegen::settings::Flags::new(cranelift_codegen::settings::builder()); |
251 | match cranelift_codegen::verify_function(&func, &flags) { | |
252 | Ok(_) => {} | |
253 | Err(err) => { | |
254 | tcx.sess.err(&format!("{:?}", err)); | |
255 | let pretty_error = cranelift_codegen::print_errors::pretty_verifier_error( | |
256 | &func, | |
29967ef6 XL |
257 | Some(Box::new(writer)), |
258 | err, | |
259 | ); | |
6a06907d | 260 | tcx.sess.fatal(&format!("cranelift verify error:\n{}", pretty_error)); |
29967ef6 XL |
261 | } |
262 | } | |
263 | }); | |
264 | } | |
265 | ||
f2b60f7d FG |
266 | fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) { |
267 | if !crate::constant::check_constants(fx) { | |
268 | fx.bcx.append_block_params_for_function_params(fx.block_map[START_BLOCK]); | |
269 | fx.bcx.switch_to_block(fx.block_map[START_BLOCK]); | |
270 | // compilation should have been aborted | |
271 | fx.bcx.ins().trap(TrapCode::UnreachableCodeReached); | |
272 | return; | |
273 | } | |
274 | ||
275 | let arg_uninhabited = fx | |
276 | .mir | |
277 | .args_iter() | |
278 | .any(|arg| fx.layout_of(fx.monomorphize(fx.mir.local_decls[arg].ty)).abi.is_uninhabited()); | |
279 | if arg_uninhabited { | |
280 | fx.bcx.append_block_params_for_function_params(fx.block_map[START_BLOCK]); | |
281 | fx.bcx.switch_to_block(fx.block_map[START_BLOCK]); | |
282 | fx.bcx.ins().trap(TrapCode::UnreachableCodeReached); | |
283 | return; | |
284 | } | |
9ffffee4 FG |
285 | fx.tcx |
286 | .prof | |
287 | .generic_activity("codegen prelude") | |
288 | .run(|| crate::abi::codegen_fn_prelude(fx, start_block)); | |
f2b60f7d FG |
289 | |
290 | for (bb, bb_data) in fx.mir.basic_blocks.iter_enumerated() { | |
29967ef6 XL |
291 | let block = fx.get_block(bb); |
292 | fx.bcx.switch_to_block(block); | |
293 | ||
294 | if bb_data.is_cleanup { | |
295 | // Unwinding after panicking is not supported | |
296 | continue; | |
297 | ||
17df50a5 XL |
298 | // FIXME Once unwinding is supported and Cranelift supports marking blocks as cold, do |
299 | // so for cleanup blocks. | |
29967ef6 XL |
300 | } |
301 | ||
302 | fx.bcx.ins().nop(); | |
303 | for stmt in &bb_data.statements { | |
304 | fx.set_debug_loc(stmt.source_info); | |
305 | codegen_stmt(fx, block, stmt); | |
306 | } | |
307 | ||
cdc7bbd5 | 308 | if fx.clif_comments.enabled() { |
29967ef6 | 309 | let mut terminator_head = "\n".to_string(); |
5e7ed085 FG |
310 | with_no_trimmed_paths!({ |
311 | bb_data.terminator().kind.fmt_head(&mut terminator_head).unwrap(); | |
312 | }); | |
29967ef6 XL |
313 | let inst = fx.bcx.func.layout.last_inst(block).unwrap(); |
314 | fx.add_comment(inst, terminator_head); | |
315 | } | |
316 | ||
a2a8927a XL |
317 | let source_info = bb_data.terminator().source_info; |
318 | fx.set_debug_loc(source_info); | |
29967ef6 | 319 | |
9ffffee4 FG |
320 | let _print_guard = |
321 | crate::PrintOnPanic(|| format!("terminator {:?}", bb_data.terminator().kind)); | |
322 | ||
29967ef6 XL |
323 | match &bb_data.terminator().kind { |
324 | TerminatorKind::Goto { target } => { | |
325 | if let TerminatorKind::Return = fx.mir[*target].terminator().kind { | |
326 | let mut can_immediately_return = true; | |
327 | for stmt in &fx.mir[*target].statements { | |
328 | if let StatementKind::StorageDead(_) = stmt.kind { | |
329 | } else { | |
330 | // FIXME Can sometimes happen, see rust-lang/rust#70531 | |
331 | can_immediately_return = false; | |
332 | break; | |
333 | } | |
334 | } | |
335 | ||
336 | if can_immediately_return { | |
337 | crate::abi::codegen_return(fx); | |
338 | continue; | |
339 | } | |
340 | } | |
341 | ||
342 | let block = fx.get_block(*target); | |
343 | fx.bcx.ins().jump(block, &[]); | |
344 | } | |
345 | TerminatorKind::Return => { | |
346 | crate::abi::codegen_return(fx); | |
347 | } | |
353b0b11 FG |
348 | TerminatorKind::Assert { cond, expected, msg, target, unwind: _ } => { |
349 | if !fx.tcx.sess.overflow_checks() && msg.is_optional_overflow_check() { | |
350 | let target = fx.get_block(*target); | |
351 | fx.bcx.ins().jump(target, &[]); | |
352 | continue; | |
29967ef6 XL |
353 | } |
354 | let cond = codegen_operand(fx, cond).load_scalar(fx); | |
355 | ||
356 | let target = fx.get_block(*target); | |
357 | let failure = fx.bcx.create_block(); | |
5e7ed085 | 358 | fx.bcx.set_cold_block(failure); |
29967ef6 XL |
359 | |
360 | if *expected { | |
353b0b11 | 361 | fx.bcx.ins().brif(cond, target, &[], failure, &[]); |
29967ef6 | 362 | } else { |
353b0b11 | 363 | fx.bcx.ins().brif(cond, failure, &[], target, &[]); |
29967ef6 | 364 | }; |
29967ef6 XL |
365 | |
366 | fx.bcx.switch_to_block(failure); | |
367 | fx.bcx.ins().nop(); | |
368 | ||
369 | match msg { | |
370 | AssertKind::BoundsCheck { ref len, ref index } => { | |
371 | let len = codegen_operand(fx, len).load_scalar(fx); | |
372 | let index = codegen_operand(fx, index).load_scalar(fx); | |
923072b8 | 373 | let location = fx.get_caller_location(source_info).load_scalar(fx); |
29967ef6 XL |
374 | |
375 | codegen_panic_inner( | |
376 | fx, | |
377 | rustc_hir::LangItem::PanicBoundsCheck, | |
378 | &[index, len, location], | |
a2a8927a | 379 | source_info.span, |
29967ef6 XL |
380 | ); |
381 | } | |
353b0b11 FG |
382 | AssertKind::MisalignedPointerDereference { ref required, ref found } => { |
383 | let required = codegen_operand(fx, required).load_scalar(fx); | |
384 | let found = codegen_operand(fx, found).load_scalar(fx); | |
385 | let location = fx.get_caller_location(source_info).load_scalar(fx); | |
386 | ||
387 | codegen_panic_inner( | |
388 | fx, | |
389 | rustc_hir::LangItem::PanicBoundsCheck, | |
390 | &[required, found, location], | |
391 | source_info.span, | |
392 | ); | |
393 | } | |
29967ef6 XL |
394 | _ => { |
395 | let msg_str = msg.description(); | |
923072b8 | 396 | codegen_panic(fx, msg_str, source_info); |
29967ef6 XL |
397 | } |
398 | } | |
399 | } | |
400 | ||
9c376795 FG |
401 | TerminatorKind::SwitchInt { discr, targets } => { |
402 | let discr = codegen_operand(fx, discr); | |
403 | let switch_ty = discr.layout().ty; | |
404 | let discr = discr.load_scalar(fx); | |
29967ef6 | 405 | |
5869c6ff XL |
406 | let use_bool_opt = switch_ty.kind() == fx.tcx.types.bool.kind() |
407 | || (targets.iter().count() == 1 && targets.iter().next().unwrap().0 == 0); | |
408 | if use_bool_opt { | |
29967ef6 XL |
409 | assert_eq!(targets.iter().count(), 1); |
410 | let (then_value, then_block) = targets.iter().next().unwrap(); | |
411 | let then_block = fx.get_block(then_block); | |
412 | let else_block = fx.get_block(targets.otherwise()); | |
413 | let test_zero = match then_value { | |
414 | 0 => true, | |
415 | 1 => false, | |
416 | _ => unreachable!("{:?}", targets), | |
417 | }; | |
418 | ||
29967ef6 XL |
419 | let (discr, is_inverted) = |
420 | crate::optimize::peephole::maybe_unwrap_bool_not(&mut fx.bcx, discr); | |
421 | let test_zero = if is_inverted { !test_zero } else { test_zero }; | |
5869c6ff XL |
422 | if let Some(taken) = crate::optimize::peephole::maybe_known_branch_taken( |
423 | &fx.bcx, discr, test_zero, | |
424 | ) { | |
425 | if taken { | |
426 | fx.bcx.ins().jump(then_block, &[]); | |
427 | } else { | |
428 | fx.bcx.ins().jump(else_block, &[]); | |
429 | } | |
29967ef6 | 430 | } else { |
5869c6ff | 431 | if test_zero { |
353b0b11 | 432 | fx.bcx.ins().brif(discr, else_block, &[], then_block, &[]); |
5869c6ff | 433 | } else { |
353b0b11 | 434 | fx.bcx.ins().brif(discr, then_block, &[], else_block, &[]); |
5869c6ff | 435 | } |
29967ef6 XL |
436 | } |
437 | } else { | |
438 | let mut switch = ::cranelift_frontend::Switch::new(); | |
439 | for (value, block) in targets.iter() { | |
440 | let block = fx.get_block(block); | |
441 | switch.set_entry(value, block); | |
442 | } | |
443 | let otherwise_block = fx.get_block(targets.otherwise()); | |
444 | switch.emit(&mut fx.bcx, discr, otherwise_block); | |
445 | } | |
446 | } | |
447 | TerminatorKind::Call { | |
448 | func, | |
449 | args, | |
450 | destination, | |
923072b8 | 451 | target, |
29967ef6 | 452 | fn_span, |
353b0b11 | 453 | unwind: _, |
29967ef6 XL |
454 | from_hir_call: _, |
455 | } => { | |
9ffffee4 | 456 | fx.tcx.prof.generic_activity("codegen call").run(|| { |
923072b8 FG |
457 | crate::abi::codegen_terminator_call( |
458 | fx, | |
459 | mir::SourceInfo { span: *fn_span, ..source_info }, | |
460 | func, | |
461 | args, | |
462 | *destination, | |
463 | *target, | |
464 | ) | |
29967ef6 XL |
465 | }); |
466 | } | |
467 | TerminatorKind::InlineAsm { | |
468 | template, | |
469 | operands, | |
470 | options, | |
471 | destination, | |
472 | line_spans: _, | |
353b0b11 | 473 | unwind: _, |
29967ef6 | 474 | } => { |
a2a8927a XL |
475 | if options.contains(InlineAsmOptions::MAY_UNWIND) { |
476 | fx.tcx.sess.span_fatal( | |
477 | source_info.span, | |
478 | "cranelift doesn't support unwinding from inline assembly.", | |
479 | ); | |
480 | } | |
481 | ||
29967ef6 XL |
482 | crate::inline_asm::codegen_inline_asm( |
483 | fx, | |
a2a8927a | 484 | source_info.span, |
29967ef6 XL |
485 | template, |
486 | operands, | |
487 | *options, | |
f2b60f7d | 488 | *destination, |
29967ef6 | 489 | ); |
29967ef6 | 490 | } |
353b0b11 | 491 | TerminatorKind::Terminate => { |
9ffffee4 FG |
492 | codegen_panic_cannot_unwind(fx, source_info); |
493 | } | |
494 | TerminatorKind::Resume => { | |
5e7ed085 FG |
495 | // FIXME implement unwinding |
496 | fx.bcx.ins().trap(TrapCode::UnreachableCodeReached); | |
29967ef6 XL |
497 | } |
498 | TerminatorKind::Unreachable => { | |
5e7ed085 | 499 | fx.bcx.ins().trap(TrapCode::UnreachableCodeReached); |
29967ef6 XL |
500 | } |
501 | TerminatorKind::Yield { .. } | |
502 | | TerminatorKind::FalseEdge { .. } | |
503 | | TerminatorKind::FalseUnwind { .. } | |
29967ef6 XL |
504 | | TerminatorKind::GeneratorDrop => { |
505 | bug!("shouldn't exist at codegen {:?}", bb_data.terminator()); | |
506 | } | |
6a06907d | 507 | TerminatorKind::Drop { place, target, unwind: _ } => { |
29967ef6 | 508 | let drop_place = codegen_place(fx, *place); |
923072b8 | 509 | crate::abi::codegen_drop(fx, source_info, drop_place); |
29967ef6 XL |
510 | |
511 | let target_block = fx.get_block(*target); | |
512 | fx.bcx.ins().jump(target_block, &[]); | |
513 | } | |
514 | }; | |
515 | } | |
29967ef6 XL |
516 | } |
517 | ||
518 | fn codegen_stmt<'tcx>( | |
6a06907d | 519 | fx: &mut FunctionCx<'_, '_, 'tcx>, |
29967ef6 XL |
520 | #[allow(unused_variables)] cur_block: Block, |
521 | stmt: &Statement<'tcx>, | |
522 | ) { | |
523 | let _print_guard = crate::PrintOnPanic(|| format!("stmt {:?}", stmt)); | |
524 | ||
525 | fx.set_debug_loc(stmt.source_info); | |
526 | ||
923072b8 | 527 | #[cfg(any())] // This is never true |
29967ef6 XL |
528 | match &stmt.kind { |
529 | StatementKind::StorageLive(..) | StatementKind::StorageDead(..) => {} // Those are not very useful | |
530 | _ => { | |
cdc7bbd5 XL |
531 | if fx.clif_comments.enabled() { |
532 | let inst = fx.bcx.func.layout.last_inst(cur_block).unwrap(); | |
533 | fx.add_comment(inst, format!("{:?}", stmt)); | |
534 | } | |
29967ef6 XL |
535 | } |
536 | } | |
537 | ||
538 | match &stmt.kind { | |
6a06907d | 539 | StatementKind::SetDiscriminant { place, variant_index } => { |
29967ef6 XL |
540 | let place = codegen_place(fx, **place); |
541 | crate::discriminant::codegen_set_discriminant(fx, place, *variant_index); | |
542 | } | |
543 | StatementKind::Assign(to_place_and_rval) => { | |
544 | let lval = codegen_place(fx, to_place_and_rval.0); | |
545 | let dest_layout = lval.layout(); | |
fc512014 XL |
546 | match to_place_and_rval.1 { |
547 | Rvalue::Use(ref operand) => { | |
29967ef6 XL |
548 | let val = codegen_operand(fx, operand); |
549 | lval.write_cvalue(fx, val); | |
550 | } | |
064997fb FG |
551 | Rvalue::CopyForDeref(place) => { |
552 | let cplace = codegen_place(fx, place); | |
553 | let val = cplace.to_cvalue(fx); | |
554 | lval.write_cvalue(fx, val) | |
555 | } | |
29967ef6 | 556 | Rvalue::Ref(_, _, place) | Rvalue::AddressOf(_, place) => { |
fc512014 | 557 | let place = codegen_place(fx, place); |
29967ef6 XL |
558 | let ref_ = place.place_ref(fx, lval.layout()); |
559 | lval.write_cvalue(fx, ref_); | |
560 | } | |
561 | Rvalue::ThreadLocalRef(def_id) => { | |
fc512014 | 562 | let val = crate::constant::codegen_tls_ref(fx, def_id, lval.layout()); |
29967ef6 XL |
563 | lval.write_cvalue(fx, val); |
564 | } | |
cdc7bbd5 XL |
565 | Rvalue::BinaryOp(bin_op, ref lhs_rhs) => { |
566 | let lhs = codegen_operand(fx, &lhs_rhs.0); | |
567 | let rhs = codegen_operand(fx, &lhs_rhs.1); | |
29967ef6 | 568 | |
fc512014 | 569 | let res = crate::num::codegen_binop(fx, bin_op, lhs, rhs); |
29967ef6 XL |
570 | lval.write_cvalue(fx, res); |
571 | } | |
cdc7bbd5 XL |
572 | Rvalue::CheckedBinaryOp(bin_op, ref lhs_rhs) => { |
573 | let lhs = codegen_operand(fx, &lhs_rhs.0); | |
574 | let rhs = codegen_operand(fx, &lhs_rhs.1); | |
29967ef6 | 575 | |
9ffffee4 | 576 | let res = crate::num::codegen_checked_int_binop(fx, bin_op, lhs, rhs); |
29967ef6 XL |
577 | lval.write_cvalue(fx, res); |
578 | } | |
fc512014 | 579 | Rvalue::UnaryOp(un_op, ref operand) => { |
29967ef6 XL |
580 | let operand = codegen_operand(fx, operand); |
581 | let layout = operand.layout(); | |
582 | let val = operand.load_scalar(fx); | |
583 | let res = match un_op { | |
584 | UnOp::Not => match layout.ty.kind() { | |
585 | ty::Bool => { | |
586 | let res = fx.bcx.ins().icmp_imm(IntCC::Equal, val, 0); | |
9c376795 | 587 | CValue::by_val(res, layout) |
29967ef6 XL |
588 | } |
589 | ty::Uint(_) | ty::Int(_) => { | |
590 | CValue::by_val(fx.bcx.ins().bnot(val), layout) | |
591 | } | |
592 | _ => unreachable!("un op Not for {:?}", layout.ty), | |
593 | }, | |
594 | UnOp::Neg => match layout.ty.kind() { | |
29967ef6 XL |
595 | ty::Int(_) => CValue::by_val(fx.bcx.ins().ineg(val), layout), |
596 | ty::Float(_) => CValue::by_val(fx.bcx.ins().fneg(val), layout), | |
597 | _ => unreachable!("un op Neg for {:?}", layout.ty), | |
598 | }, | |
599 | }; | |
600 | lval.write_cvalue(fx, res); | |
601 | } | |
fc512014 XL |
602 | Rvalue::Cast( |
603 | CastKind::Pointer(PointerCast::ReifyFnPointer), | |
604 | ref operand, | |
605 | to_ty, | |
606 | ) => { | |
607 | let from_ty = fx.monomorphize(operand.ty(&fx.mir.local_decls, fx.tcx)); | |
29967ef6 XL |
608 | let to_layout = fx.layout_of(fx.monomorphize(to_ty)); |
609 | match *from_ty.kind() { | |
610 | ty::FnDef(def_id, substs) => { | |
611 | let func_ref = fx.get_function_ref( | |
612 | Instance::resolve_for_fn_ptr( | |
613 | fx.tcx, | |
614 | ParamEnv::reveal_all(), | |
615 | def_id, | |
616 | substs, | |
617 | ) | |
618 | .unwrap() | |
619 | .polymorphize(fx.tcx), | |
620 | ); | |
621 | let func_addr = fx.bcx.ins().func_addr(fx.pointer_type, func_ref); | |
622 | lval.write_cvalue(fx, CValue::by_val(func_addr, to_layout)); | |
623 | } | |
624 | _ => bug!("Trying to ReifyFnPointer on non FnDef {:?}", from_ty), | |
625 | } | |
626 | } | |
fc512014 XL |
627 | Rvalue::Cast( |
628 | CastKind::Pointer(PointerCast::UnsafeFnPointer), | |
629 | ref operand, | |
630 | to_ty, | |
631 | ) | |
632 | | Rvalue::Cast( | |
633 | CastKind::Pointer(PointerCast::MutToConstPointer), | |
634 | ref operand, | |
635 | to_ty, | |
636 | ) | |
637 | | Rvalue::Cast( | |
638 | CastKind::Pointer(PointerCast::ArrayToPointer), | |
639 | ref operand, | |
640 | to_ty, | |
641 | ) => { | |
29967ef6 XL |
642 | let to_layout = fx.layout_of(fx.monomorphize(to_ty)); |
643 | let operand = codegen_operand(fx, operand); | |
644 | lval.write_cvalue(fx, operand.cast_pointer_to(to_layout)); | |
645 | } | |
923072b8 | 646 | Rvalue::Cast( |
2b03887a FG |
647 | CastKind::IntToInt |
648 | | CastKind::FloatToFloat | |
649 | | CastKind::FloatToInt | |
650 | | CastKind::IntToFloat | |
651 | | CastKind::FnPtrToPtr | |
652 | | CastKind::PtrToPtr | |
923072b8 FG |
653 | | CastKind::PointerExposeAddress |
654 | | CastKind::PointerFromExposedAddress, | |
655 | ref operand, | |
656 | to_ty, | |
657 | ) => { | |
29967ef6 XL |
658 | let operand = codegen_operand(fx, operand); |
659 | let from_ty = operand.layout().ty; | |
660 | let to_ty = fx.monomorphize(to_ty); | |
661 | ||
6a06907d | 662 | fn is_fat_ptr<'tcx>(fx: &FunctionCx<'_, '_, 'tcx>, ty: Ty<'tcx>) -> bool { |
29967ef6 | 663 | ty.builtin_deref(true) |
6a06907d XL |
664 | .map(|ty::TypeAndMut { ty: pointee_ty, mutbl: _ }| { |
665 | has_ptr_meta(fx.tcx, pointee_ty) | |
666 | }) | |
29967ef6 XL |
667 | .unwrap_or(false) |
668 | } | |
669 | ||
670 | if is_fat_ptr(fx, from_ty) { | |
671 | if is_fat_ptr(fx, to_ty) { | |
672 | // fat-ptr -> fat-ptr | |
673 | lval.write_cvalue(fx, operand.cast_pointer_to(dest_layout)); | |
674 | } else { | |
675 | // fat-ptr -> thin-ptr | |
676 | let (ptr, _extra) = operand.load_scalar_pair(fx); | |
677 | lval.write_cvalue(fx, CValue::by_val(ptr, dest_layout)) | |
678 | } | |
29967ef6 XL |
679 | } else { |
680 | let to_clif_ty = fx.clif_type(to_ty).unwrap(); | |
681 | let from = operand.load_scalar(fx); | |
682 | ||
683 | let res = clif_int_or_float_cast( | |
684 | fx, | |
685 | from, | |
686 | type_sign(from_ty), | |
687 | to_clif_ty, | |
688 | type_sign(to_ty), | |
689 | ); | |
690 | lval.write_cvalue(fx, CValue::by_val(res, dest_layout)); | |
691 | } | |
692 | } | |
693 | Rvalue::Cast( | |
694 | CastKind::Pointer(PointerCast::ClosureFnPointer(_)), | |
fc512014 | 695 | ref operand, |
29967ef6 XL |
696 | _to_ty, |
697 | ) => { | |
698 | let operand = codegen_operand(fx, operand); | |
699 | match *operand.layout().ty.kind() { | |
700 | ty::Closure(def_id, substs) => { | |
701 | let instance = Instance::resolve_closure( | |
702 | fx.tcx, | |
703 | def_id, | |
704 | substs, | |
705 | ty::ClosureKind::FnOnce, | |
706 | ) | |
064997fb | 707 | .expect("failed to normalize and resolve closure during codegen") |
29967ef6 XL |
708 | .polymorphize(fx.tcx); |
709 | let func_ref = fx.get_function_ref(instance); | |
710 | let func_addr = fx.bcx.ins().func_addr(fx.pointer_type, func_ref); | |
711 | lval.write_cvalue(fx, CValue::by_val(func_addr, lval.layout())); | |
712 | } | |
713 | _ => bug!("{} cannot be cast to a fn ptr", operand.layout().ty), | |
714 | } | |
715 | } | |
fc512014 | 716 | Rvalue::Cast(CastKind::Pointer(PointerCast::Unsize), ref operand, _to_ty) => { |
29967ef6 XL |
717 | let operand = codegen_operand(fx, operand); |
718 | operand.unsize_value(fx, lval); | |
719 | } | |
2b03887a FG |
720 | Rvalue::Cast(CastKind::DynStar, ref operand, _) => { |
721 | let operand = codegen_operand(fx, operand); | |
722 | operand.coerce_dyn_star(fx, lval); | |
f2b60f7d | 723 | } |
353b0b11 FG |
724 | Rvalue::Cast(CastKind::Transmute, ref operand, _to_ty) => { |
725 | let operand = codegen_operand(fx, operand); | |
726 | lval.write_cvalue_transmute(fx, operand); | |
727 | } | |
29967ef6 | 728 | Rvalue::Discriminant(place) => { |
fc512014 | 729 | let place = codegen_place(fx, place); |
29967ef6 | 730 | let value = place.to_cvalue(fx); |
f2b60f7d | 731 | crate::discriminant::codegen_get_discriminant(fx, lval, value, dest_layout); |
29967ef6 | 732 | } |
fc512014 | 733 | Rvalue::Repeat(ref operand, times) => { |
29967ef6 XL |
734 | let operand = codegen_operand(fx, operand); |
735 | let times = fx | |
736 | .monomorphize(times) | |
737 | .eval(fx.tcx, ParamEnv::reveal_all()) | |
923072b8 | 738 | .kind() |
29967ef6 XL |
739 | .try_to_bits(fx.tcx.data_layout.pointer_size) |
740 | .unwrap(); | |
cdc7bbd5 XL |
741 | if operand.layout().size.bytes() == 0 { |
742 | // Do nothing for ZST's | |
743 | } else if fx.clif_type(operand.layout().ty) == Some(types::I8) { | |
29967ef6 XL |
744 | let times = fx.bcx.ins().iconst(fx.pointer_type, times as i64); |
745 | // FIXME use emit_small_memset where possible | |
746 | let addr = lval.to_ptr().get_addr(fx); | |
747 | let val = operand.load_scalar(fx); | |
a2a8927a | 748 | fx.bcx.call_memset(fx.target_config, addr, val, times); |
29967ef6 XL |
749 | } else { |
750 | let loop_block = fx.bcx.create_block(); | |
751 | let loop_block2 = fx.bcx.create_block(); | |
752 | let done_block = fx.bcx.create_block(); | |
753 | let index = fx.bcx.append_block_param(loop_block, fx.pointer_type); | |
754 | let zero = fx.bcx.ins().iconst(fx.pointer_type, 0); | |
755 | fx.bcx.ins().jump(loop_block, &[zero]); | |
756 | ||
757 | fx.bcx.switch_to_block(loop_block); | |
758 | let done = fx.bcx.ins().icmp_imm(IntCC::Equal, index, times as i64); | |
353b0b11 | 759 | fx.bcx.ins().brif(done, done_block, &[], loop_block2, &[]); |
29967ef6 XL |
760 | |
761 | fx.bcx.switch_to_block(loop_block2); | |
762 | let to = lval.place_index(fx, index); | |
763 | to.write_cvalue(fx, operand); | |
764 | let index = fx.bcx.ins().iadd_imm(index, 1); | |
765 | fx.bcx.ins().jump(loop_block, &[index]); | |
766 | ||
767 | fx.bcx.switch_to_block(done_block); | |
768 | fx.bcx.ins().nop(); | |
769 | } | |
770 | } | |
771 | Rvalue::Len(place) => { | |
fc512014 | 772 | let place = codegen_place(fx, place); |
29967ef6 XL |
773 | let usize_layout = fx.layout_of(fx.tcx.types.usize); |
774 | let len = codegen_array_len(fx, place); | |
775 | lval.write_cvalue(fx, CValue::by_val(len, usize_layout)); | |
776 | } | |
c295e0f8 XL |
777 | Rvalue::ShallowInitBox(ref operand, content_ty) => { |
778 | let content_ty = fx.monomorphize(content_ty); | |
779 | let box_layout = fx.layout_of(fx.tcx.mk_box(content_ty)); | |
780 | let operand = codegen_operand(fx, operand); | |
781 | let operand = operand.load_scalar(fx); | |
782 | lval.write_cvalue(fx, CValue::by_val(operand, box_layout)); | |
783 | } | |
c295e0f8 | 784 | Rvalue::NullaryOp(null_op, ty) => { |
2b03887a | 785 | assert!(lval.layout().ty.is_sized(fx.tcx, ParamEnv::reveal_all())); |
c295e0f8 XL |
786 | let layout = fx.layout_of(fx.monomorphize(ty)); |
787 | let val = match null_op { | |
788 | NullOp::SizeOf => layout.size.bytes(), | |
789 | NullOp::AlignOf => layout.align.abi.bytes(), | |
c295e0f8 | 790 | }; |
a2a8927a | 791 | let val = CValue::const_val(fx, fx.layout_of(fx.tcx.types.usize), val.into()); |
29967ef6 XL |
792 | lval.write_cvalue(fx, val); |
793 | } | |
9ffffee4 FG |
794 | Rvalue::Aggregate(ref kind, ref operands) => { |
795 | let (variant_index, variant_dest, active_field_index) = match **kind { | |
796 | mir::AggregateKind::Adt(_, variant_index, _, _, active_field_index) => { | |
797 | let variant_dest = lval.downcast_variant(fx, variant_index); | |
798 | (variant_index, variant_dest, active_field_index) | |
29967ef6 | 799 | } |
353b0b11 | 800 | _ => (FIRST_VARIANT, lval, None), |
9ffffee4 FG |
801 | }; |
802 | if active_field_index.is_some() { | |
803 | assert_eq!(operands.len(), 1); | |
29967ef6 | 804 | } |
353b0b11 | 805 | for (i, operand) in operands.iter_enumerated() { |
9ffffee4 FG |
806 | let operand = codegen_operand(fx, operand); |
807 | let field_index = active_field_index.unwrap_or(i); | |
808 | let to = if let mir::AggregateKind::Array(_) = **kind { | |
353b0b11 FG |
809 | let array_index = i64::from(field_index.as_u32()); |
810 | let index = fx.bcx.ins().iconst(fx.pointer_type, array_index); | |
9ffffee4 FG |
811 | variant_dest.place_index(fx, index) |
812 | } else { | |
353b0b11 | 813 | variant_dest.place_field(fx, field_index) |
9ffffee4 FG |
814 | }; |
815 | to.write_cvalue(fx, operand); | |
816 | } | |
817 | crate::discriminant::codegen_set_discriminant(fx, lval, variant_index); | |
818 | } | |
29967ef6 XL |
819 | } |
820 | } | |
821 | StatementKind::StorageLive(_) | |
822 | | StatementKind::StorageDead(_) | |
04454e1e | 823 | | StatementKind::Deinit(_) |
9ffffee4 | 824 | | StatementKind::ConstEvalCounter |
29967ef6 XL |
825 | | StatementKind::Nop |
826 | | StatementKind::FakeRead(..) | |
827 | | StatementKind::Retag { .. } | |
353b0b11 | 828 | | StatementKind::PlaceMention(..) |
29967ef6 XL |
829 | | StatementKind::AscribeUserType(..) => {} |
830 | ||
29967ef6 | 831 | StatementKind::Coverage { .. } => fx.tcx.sess.fatal("-Zcoverage is unimplemented"), |
f2b60f7d FG |
832 | StatementKind::Intrinsic(ref intrinsic) => match &**intrinsic { |
833 | // We ignore `assume` intrinsics, they are only useful for optimizations | |
834 | NonDivergingIntrinsic::Assume(_) => {} | |
835 | NonDivergingIntrinsic::CopyNonOverlapping(mir::CopyNonOverlapping { | |
836 | src, | |
837 | dst, | |
838 | count, | |
839 | }) => { | |
840 | let dst = codegen_operand(fx, dst); | |
841 | let pointee = dst | |
842 | .layout() | |
843 | .pointee_info_at(fx, rustc_target::abi::Size::ZERO) | |
844 | .expect("Expected pointer"); | |
845 | let dst = dst.load_scalar(fx); | |
846 | let src = codegen_operand(fx, src).load_scalar(fx); | |
847 | let count = codegen_operand(fx, count).load_scalar(fx); | |
848 | let elem_size: u64 = pointee.size.bytes(); | |
849 | let bytes = if elem_size != 1 { | |
850 | fx.bcx.ins().imul_imm(count, elem_size as i64) | |
851 | } else { | |
852 | count | |
853 | }; | |
854 | fx.bcx.call_memcpy(fx.target_config, dst, src, bytes); | |
855 | } | |
856 | }, | |
29967ef6 XL |
857 | } |
858 | } | |
859 | ||
6a06907d | 860 | fn codegen_array_len<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, place: CPlace<'tcx>) -> Value { |
29967ef6 XL |
861 | match *place.layout().ty.kind() { |
862 | ty::Array(_elem_ty, len) => { | |
9ffffee4 | 863 | let len = fx.monomorphize(len).eval_target_usize(fx.tcx, ParamEnv::reveal_all()) as i64; |
29967ef6 XL |
864 | fx.bcx.ins().iconst(fx.pointer_type, len) |
865 | } | |
6a06907d XL |
866 | ty::Slice(_elem_ty) => { |
867 | place.to_ptr_maybe_unsized().1.expect("Length metadata for slice place") | |
868 | } | |
29967ef6 XL |
869 | _ => bug!("Rvalue::Len({:?})", place), |
870 | } | |
871 | } | |
872 | ||
873 | pub(crate) fn codegen_place<'tcx>( | |
6a06907d | 874 | fx: &mut FunctionCx<'_, '_, 'tcx>, |
29967ef6 XL |
875 | place: Place<'tcx>, |
876 | ) -> CPlace<'tcx> { | |
877 | let mut cplace = fx.get_local_place(place.local); | |
878 | ||
879 | for elem in place.projection { | |
880 | match elem { | |
881 | PlaceElem::Deref => { | |
064997fb | 882 | cplace = cplace.place_deref(fx); |
29967ef6 | 883 | } |
2b03887a | 884 | PlaceElem::OpaqueCast(ty) => cplace = cplace.place_opaque_cast(fx, ty), |
29967ef6 XL |
885 | PlaceElem::Field(field, _ty) => { |
886 | cplace = cplace.place_field(fx, field); | |
887 | } | |
888 | PlaceElem::Index(local) => { | |
889 | let index = fx.get_local_place(local).to_cvalue(fx).load_scalar(fx); | |
890 | cplace = cplace.place_index(fx, index); | |
891 | } | |
6a06907d | 892 | PlaceElem::ConstantIndex { offset, min_length: _, from_end } => { |
29967ef6 XL |
893 | let offset: u64 = offset; |
894 | let index = if !from_end { | |
895 | fx.bcx.ins().iconst(fx.pointer_type, offset as i64) | |
896 | } else { | |
897 | let len = codegen_array_len(fx, cplace); | |
898 | fx.bcx.ins().iadd_imm(len, -(offset as i64)) | |
899 | }; | |
900 | cplace = cplace.place_index(fx, index); | |
901 | } | |
902 | PlaceElem::Subslice { from, to, from_end } => { | |
903 | // These indices are generated by slice patterns. | |
904 | // slice[from:-to] in Python terms. | |
905 | ||
906 | let from: u64 = from; | |
907 | let to: u64 = to; | |
908 | ||
909 | match cplace.layout().ty.kind() { | |
910 | ty::Array(elem_ty, _len) => { | |
911 | assert!(!from_end, "array subslices are never `from_end`"); | |
5099ac24 | 912 | let elem_layout = fx.layout_of(*elem_ty); |
29967ef6 XL |
913 | let ptr = cplace.to_ptr(); |
914 | cplace = CPlace::for_ptr( | |
915 | ptr.offset_i64(fx, elem_layout.size.bytes() as i64 * (from as i64)), | |
5099ac24 | 916 | fx.layout_of(fx.tcx.mk_array(*elem_ty, to - from)), |
29967ef6 XL |
917 | ); |
918 | } | |
919 | ty::Slice(elem_ty) => { | |
920 | assert!(from_end, "slice subslices should be `from_end`"); | |
5099ac24 | 921 | let elem_layout = fx.layout_of(*elem_ty); |
29967ef6 XL |
922 | let (ptr, len) = cplace.to_ptr_maybe_unsized(); |
923 | let len = len.unwrap(); | |
924 | cplace = CPlace::for_ptr_with_extra( | |
925 | ptr.offset_i64(fx, elem_layout.size.bytes() as i64 * (from as i64)), | |
926 | fx.bcx.ins().iadd_imm(len, -(from as i64 + to as i64)), | |
927 | cplace.layout(), | |
928 | ); | |
929 | } | |
930 | _ => unreachable!(), | |
931 | } | |
932 | } | |
933 | PlaceElem::Downcast(_adt_def, variant) => { | |
934 | cplace = cplace.downcast_variant(fx, variant); | |
935 | } | |
936 | } | |
937 | } | |
938 | ||
939 | cplace | |
940 | } | |
941 | ||
942 | pub(crate) fn codegen_operand<'tcx>( | |
6a06907d | 943 | fx: &mut FunctionCx<'_, '_, 'tcx>, |
29967ef6 XL |
944 | operand: &Operand<'tcx>, |
945 | ) -> CValue<'tcx> { | |
946 | match operand { | |
947 | Operand::Move(place) | Operand::Copy(place) => { | |
948 | let cplace = codegen_place(fx, *place); | |
949 | cplace.to_cvalue(fx) | |
950 | } | |
2b03887a | 951 | Operand::Constant(const_) => crate::constant::codegen_constant_operand(fx, const_), |
29967ef6 XL |
952 | } |
953 | } | |
954 | ||
923072b8 FG |
955 | pub(crate) fn codegen_panic<'tcx>( |
956 | fx: &mut FunctionCx<'_, '_, 'tcx>, | |
957 | msg_str: &str, | |
958 | source_info: mir::SourceInfo, | |
959 | ) { | |
960 | let location = fx.get_caller_location(source_info).load_scalar(fx); | |
29967ef6 | 961 | |
17df50a5 | 962 | let msg_ptr = fx.anonymous_str(msg_str); |
6a06907d | 963 | let msg_len = fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(msg_str.len()).unwrap()); |
29967ef6 XL |
964 | let args = [msg_ptr, msg_len, location]; |
965 | ||
923072b8 | 966 | codegen_panic_inner(fx, rustc_hir::LangItem::Panic, &args, source_info.span); |
29967ef6 XL |
967 | } |
968 | ||
9ffffee4 FG |
969 | pub(crate) fn codegen_panic_nounwind<'tcx>( |
970 | fx: &mut FunctionCx<'_, '_, 'tcx>, | |
971 | msg_str: &str, | |
972 | source_info: mir::SourceInfo, | |
973 | ) { | |
974 | let msg_ptr = fx.anonymous_str(msg_str); | |
975 | let msg_len = fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(msg_str.len()).unwrap()); | |
976 | let args = [msg_ptr, msg_len]; | |
977 | ||
978 | codegen_panic_inner(fx, rustc_hir::LangItem::PanicNounwind, &args, source_info.span); | |
979 | } | |
980 | ||
981 | pub(crate) fn codegen_panic_cannot_unwind<'tcx>( | |
982 | fx: &mut FunctionCx<'_, '_, 'tcx>, | |
983 | source_info: mir::SourceInfo, | |
984 | ) { | |
985 | let args = []; | |
986 | ||
987 | codegen_panic_inner(fx, rustc_hir::LangItem::PanicCannotUnwind, &args, source_info.span); | |
988 | } | |
989 | ||
990 | fn codegen_panic_inner<'tcx>( | |
6a06907d | 991 | fx: &mut FunctionCx<'_, '_, 'tcx>, |
29967ef6 XL |
992 | lang_item: rustc_hir::LangItem, |
993 | args: &[Value], | |
994 | span: Span, | |
995 | ) { | |
f2b60f7d FG |
996 | let def_id = fx |
997 | .tcx | |
998 | .lang_items() | |
999 | .require(lang_item) | |
1000 | .unwrap_or_else(|e| fx.tcx.sess.span_fatal(span, e.to_string())); | |
29967ef6 XL |
1001 | |
1002 | let instance = Instance::mono(fx.tcx, def_id).polymorphize(fx.tcx); | |
1003 | let symbol_name = fx.tcx.symbol_name(instance).name; | |
1004 | ||
1005 | fx.lib_call( | |
353b0b11 | 1006 | symbol_name, |
9ffffee4 | 1007 | args.iter().map(|&arg| AbiParam::new(fx.bcx.func.dfg.value_type(arg))).collect(), |
29967ef6 XL |
1008 | vec![], |
1009 | args, | |
1010 | ); | |
1011 | ||
5e7ed085 | 1012 | fx.bcx.ins().trap(TrapCode::UnreachableCodeReached); |
29967ef6 | 1013 | } |