]>
Commit | Line | Data |
---|---|---|
29967ef6 XL |
1 | //! Codegen of a single function |
2 | ||
a2a8927a | 3 | use rustc_ast::InlineAsmOptions; |
49aad941 | 4 | use rustc_index::IndexVec; |
fe692bf9 | 5 | use rustc_middle::ty::adjustment::PointerCoercion; |
c295e0f8 | 6 | use rustc_middle::ty::layout::FnAbiOf; |
5e7ed085 FG |
7 | use rustc_middle::ty::print::with_no_trimmed_paths; |
8 | ||
2b03887a FG |
9 | use cranelift_codegen::ir::UserFuncName; |
10 | ||
17df50a5 | 11 | use crate::constant::ConstantCx; |
f2b60f7d | 12 | use crate::debuginfo::FunctionDebugContext; |
29967ef6 | 13 | use crate::prelude::*; |
5e7ed085 | 14 | use crate::pretty_clif::CommentWriter; |
29967ef6 | 15 | |
f2b60f7d FG |
16 | pub(crate) struct CodegenedFunction { |
17 | symbol_name: String, | |
18 | func_id: FuncId, | |
19 | func: Function, | |
20 | clif_comments: CommentWriter, | |
21 | func_debug_cx: Option<FunctionDebugContext>, | |
22 | } | |
23 | ||
f2b60f7d FG |
24 | pub(crate) fn codegen_fn<'tcx>( |
25 | tcx: TyCtxt<'tcx>, | |
26 | cx: &mut crate::CodegenCx, | |
27 | cached_func: Function, | |
28 | module: &mut dyn Module, | |
29 | instance: Instance<'tcx>, | |
30 | ) -> CodegenedFunction { | |
49aad941 | 31 | debug_assert!(!instance.substs.has_infer()); |
fc512014 | 32 | |
9ffffee4 FG |
33 | let symbol_name = tcx.symbol_name(instance).name.to_string(); |
34 | let _timer = tcx.prof.generic_activity_with_arg("codegen fn", &*symbol_name); | |
35 | ||
29967ef6 | 36 | let mir = tcx.instance_mir(instance.def); |
136023e0 XL |
37 | let _mir_guard = crate::PrintOnPanic(|| { |
38 | let mut buf = Vec::new(); | |
5e7ed085 FG |
39 | with_no_trimmed_paths!({ |
40 | rustc_middle::mir::pretty::write_mir_fn(tcx, mir, &mut |_, _| Ok(()), &mut buf) | |
41 | .unwrap(); | |
42 | }); | |
136023e0 XL |
43 | String::from_utf8_lossy(&buf).into_owned() |
44 | }); | |
29967ef6 XL |
45 | |
46 | // Declare function | |
9c376795 | 47 | let sig = get_function_sig(tcx, module.target_config().default_call_conv, instance); |
f2b60f7d | 48 | let func_id = module.declare_function(&symbol_name, Linkage::Local, &sig).unwrap(); |
29967ef6 XL |
49 | |
50 | // Make the FunctionBuilder | |
51 | let mut func_ctx = FunctionBuilderContext::new(); | |
f2b60f7d FG |
52 | let mut func = cached_func; |
53 | func.clear(); | |
2b03887a | 54 | func.name = UserFuncName::user(0, func_id.as_u32()); |
29967ef6 XL |
55 | func.signature = sig; |
56 | func.collect_debug_info(); | |
57 | ||
58 | let mut bcx = FunctionBuilder::new(&mut func, &mut func_ctx); | |
59 | ||
60 | // Predefine blocks | |
61 | let start_block = bcx.create_block(); | |
6a06907d | 62 | let block_map: IndexVec<BasicBlock, Block> = |
f2b60f7d | 63 | (0..mir.basic_blocks.len()).map(|_| bcx.create_block()).collect(); |
29967ef6 XL |
64 | |
65 | // Make FunctionCx | |
a2a8927a XL |
66 | let target_config = module.target_config(); |
67 | let pointer_type = target_config.pointer_type(); | |
29967ef6 XL |
68 | let clif_comments = crate::pretty_clif::CommentWriter::new(tcx, instance); |
69 | ||
f2b60f7d FG |
70 | let func_debug_cx = if let Some(debug_context) = &mut cx.debug_context { |
71 | Some(debug_context.define_function(tcx, &symbol_name, mir.span)) | |
72 | } else { | |
73 | None | |
74 | }; | |
75 | ||
29967ef6 XL |
76 | let mut fx = FunctionCx { |
77 | cx, | |
17df50a5 | 78 | module, |
29967ef6 | 79 | tcx, |
a2a8927a | 80 | target_config, |
29967ef6 | 81 | pointer_type, |
17df50a5 | 82 | constants_cx: ConstantCx::new(), |
f2b60f7d | 83 | func_debug_cx, |
29967ef6 XL |
84 | |
85 | instance, | |
17df50a5 | 86 | symbol_name, |
29967ef6 | 87 | mir, |
c295e0f8 | 88 | fn_abi: Some(RevealAllLayoutCx(tcx).fn_abi_of_instance(instance, ty::List::empty())), |
29967ef6 XL |
89 | |
90 | bcx, | |
91 | block_map, | |
92 | local_map: IndexVec::with_capacity(mir.local_decls.len()), | |
93 | caller_location: None, // set by `codegen_fn_prelude` | |
29967ef6 XL |
94 | |
95 | clif_comments, | |
f2b60f7d | 96 | last_source_file: None, |
29967ef6 | 97 | next_ssa_var: 0, |
29967ef6 XL |
98 | }; |
99 | ||
9ffffee4 FG |
100 | tcx.prof.generic_activity("codegen clif ir").run(|| codegen_fn_body(&mut fx, start_block)); |
101 | fx.bcx.seal_all_blocks(); | |
102 | fx.bcx.finalize(); | |
29967ef6 XL |
103 | |
104 | // Recover all necessary data from fx, before accessing func will prevent future access to it. | |
f2b60f7d | 105 | let symbol_name = fx.symbol_name; |
5e7ed085 | 106 | let clif_comments = fx.clif_comments; |
f2b60f7d | 107 | let func_debug_cx = fx.func_debug_cx; |
17df50a5 XL |
108 | |
109 | fx.constants_cx.finalize(fx.tcx, &mut *fx.module); | |
29967ef6 | 110 | |
f2b60f7d FG |
111 | if cx.should_write_ir { |
112 | crate::pretty_clif::write_clif_file( | |
113 | tcx.output_filenames(()), | |
114 | &symbol_name, | |
115 | "unopt", | |
116 | module.isa(), | |
117 | &func, | |
118 | &clif_comments, | |
119 | ); | |
120 | } | |
29967ef6 XL |
121 | |
122 | // Verify function | |
5e7ed085 FG |
123 | verify_func(tcx, &clif_comments, &func); |
124 | ||
f2b60f7d | 125 | CodegenedFunction { symbol_name, func_id, func, clif_comments, func_debug_cx } |
5e7ed085 FG |
126 | } |
127 | ||
f2b60f7d FG |
128 | pub(crate) fn compile_fn( |
129 | cx: &mut crate::CodegenCx, | |
130 | cached_context: &mut Context, | |
5e7ed085 | 131 | module: &mut dyn Module, |
f2b60f7d | 132 | codegened_func: CodegenedFunction, |
5e7ed085 | 133 | ) { |
9ffffee4 FG |
134 | let _timer = |
135 | cx.profiler.generic_activity_with_arg("compile function", &*codegened_func.symbol_name); | |
136 | ||
f2b60f7d | 137 | let clif_comments = codegened_func.clif_comments; |
5e7ed085 FG |
138 | |
139 | // Store function in context | |
f2b60f7d | 140 | let context = cached_context; |
5e7ed085 | 141 | context.clear(); |
f2b60f7d | 142 | context.func = codegened_func.func; |
29967ef6 | 143 | |
064997fb FG |
144 | #[cfg(any())] // This is never true |
145 | let _clif_guard = { | |
146 | use std::fmt::Write; | |
147 | ||
148 | let func_clone = context.func.clone(); | |
149 | let clif_comments_clone = clif_comments.clone(); | |
150 | let mut clif = String::new(); | |
151 | for flag in module.isa().flags().iter() { | |
152 | writeln!(clif, "set {}", flag).unwrap(); | |
153 | } | |
154 | write!(clif, "target {}", module.isa().triple().architecture.to_string()).unwrap(); | |
155 | for isa_flag in module.isa().isa_flags().iter() { | |
156 | write!(clif, " {}", isa_flag).unwrap(); | |
157 | } | |
158 | writeln!(clif, "\n").unwrap(); | |
fe692bf9 | 159 | writeln!(clif, "; symbol {}", codegened_func.symbol_name).unwrap(); |
064997fb FG |
160 | crate::PrintOnPanic(move || { |
161 | let mut clif = clif.clone(); | |
162 | ::cranelift_codegen::write::decorate_function( | |
163 | &mut &clif_comments_clone, | |
164 | &mut clif, | |
165 | &func_clone, | |
166 | ) | |
167 | .unwrap(); | |
168 | clif | |
169 | }) | |
170 | }; | |
171 | ||
29967ef6 | 172 | // Define function |
9ffffee4 | 173 | cx.profiler.generic_activity("define function").run(|| { |
f2b60f7d FG |
174 | context.want_disasm = cx.should_write_ir; |
175 | module.define_function(codegened_func.func_id, context).unwrap(); | |
29967ef6 XL |
176 | }); |
177 | ||
f2b60f7d FG |
178 | if cx.should_write_ir { |
179 | // Write optimized function to file for debugging | |
180 | crate::pretty_clif::write_clif_file( | |
181 | &cx.output_filenames, | |
182 | &codegened_func.symbol_name, | |
183 | "opt", | |
184 | module.isa(), | |
185 | &context.func, | |
186 | &clif_comments, | |
187 | ); | |
29967ef6 | 188 | |
49aad941 | 189 | if let Some(disasm) = &context.compiled_code().unwrap().vcode { |
f2b60f7d FG |
190 | crate::pretty_clif::write_ir_file( |
191 | &cx.output_filenames, | |
192 | &format!("{}.vcode", codegened_func.symbol_name), | |
193 | |file| file.write_all(disasm.as_bytes()), | |
194 | ) | |
195 | } | |
5869c6ff XL |
196 | } |
197 | ||
29967ef6 | 198 | // Define debuginfo for function |
17df50a5 | 199 | let isa = module.isa(); |
29967ef6 XL |
200 | let debug_context = &mut cx.debug_context; |
201 | let unwind_context = &mut cx.unwind_context; | |
9ffffee4 | 202 | cx.profiler.generic_activity("generate debug info").run(|| { |
29967ef6 | 203 | if let Some(debug_context) = debug_context { |
f2b60f7d FG |
204 | codegened_func.func_debug_cx.unwrap().finalize( |
205 | debug_context, | |
206 | codegened_func.func_id, | |
29967ef6 | 207 | context, |
29967ef6 XL |
208 | ); |
209 | } | |
f2b60f7d | 210 | unwind_context.add_function(codegened_func.func_id, &context, isa); |
29967ef6 | 211 | }); |
29967ef6 XL |
212 | } |
213 | ||
214 | pub(crate) fn verify_func( | |
215 | tcx: TyCtxt<'_>, | |
216 | writer: &crate::pretty_clif::CommentWriter, | |
217 | func: &Function, | |
218 | ) { | |
9ffffee4 | 219 | tcx.prof.generic_activity("verify clif ir").run(|| { |
29967ef6 XL |
220 | let flags = cranelift_codegen::settings::Flags::new(cranelift_codegen::settings::builder()); |
221 | match cranelift_codegen::verify_function(&func, &flags) { | |
222 | Ok(_) => {} | |
223 | Err(err) => { | |
49aad941 | 224 | tcx.sess.err(format!("{:?}", err)); |
29967ef6 XL |
225 | let pretty_error = cranelift_codegen::print_errors::pretty_verifier_error( |
226 | &func, | |
29967ef6 XL |
227 | Some(Box::new(writer)), |
228 | err, | |
229 | ); | |
49aad941 | 230 | tcx.sess.fatal(format!("cranelift verify error:\n{}", pretty_error)); |
29967ef6 XL |
231 | } |
232 | } | |
233 | }); | |
234 | } | |
235 | ||
f2b60f7d FG |
236 | fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) { |
237 | if !crate::constant::check_constants(fx) { | |
238 | fx.bcx.append_block_params_for_function_params(fx.block_map[START_BLOCK]); | |
239 | fx.bcx.switch_to_block(fx.block_map[START_BLOCK]); | |
240 | // compilation should have been aborted | |
241 | fx.bcx.ins().trap(TrapCode::UnreachableCodeReached); | |
242 | return; | |
243 | } | |
244 | ||
245 | let arg_uninhabited = fx | |
246 | .mir | |
247 | .args_iter() | |
248 | .any(|arg| fx.layout_of(fx.monomorphize(fx.mir.local_decls[arg].ty)).abi.is_uninhabited()); | |
249 | if arg_uninhabited { | |
250 | fx.bcx.append_block_params_for_function_params(fx.block_map[START_BLOCK]); | |
251 | fx.bcx.switch_to_block(fx.block_map[START_BLOCK]); | |
252 | fx.bcx.ins().trap(TrapCode::UnreachableCodeReached); | |
253 | return; | |
254 | } | |
9ffffee4 FG |
255 | fx.tcx |
256 | .prof | |
257 | .generic_activity("codegen prelude") | |
258 | .run(|| crate::abi::codegen_fn_prelude(fx, start_block)); | |
f2b60f7d FG |
259 | |
260 | for (bb, bb_data) in fx.mir.basic_blocks.iter_enumerated() { | |
29967ef6 XL |
261 | let block = fx.get_block(bb); |
262 | fx.bcx.switch_to_block(block); | |
263 | ||
264 | if bb_data.is_cleanup { | |
265 | // Unwinding after panicking is not supported | |
266 | continue; | |
267 | ||
17df50a5 XL |
268 | // FIXME Once unwinding is supported and Cranelift supports marking blocks as cold, do |
269 | // so for cleanup blocks. | |
29967ef6 XL |
270 | } |
271 | ||
272 | fx.bcx.ins().nop(); | |
273 | for stmt in &bb_data.statements { | |
274 | fx.set_debug_loc(stmt.source_info); | |
275 | codegen_stmt(fx, block, stmt); | |
276 | } | |
277 | ||
cdc7bbd5 | 278 | if fx.clif_comments.enabled() { |
29967ef6 | 279 | let mut terminator_head = "\n".to_string(); |
5e7ed085 FG |
280 | with_no_trimmed_paths!({ |
281 | bb_data.terminator().kind.fmt_head(&mut terminator_head).unwrap(); | |
282 | }); | |
29967ef6 XL |
283 | let inst = fx.bcx.func.layout.last_inst(block).unwrap(); |
284 | fx.add_comment(inst, terminator_head); | |
285 | } | |
286 | ||
a2a8927a XL |
287 | let source_info = bb_data.terminator().source_info; |
288 | fx.set_debug_loc(source_info); | |
29967ef6 | 289 | |
9ffffee4 FG |
290 | let _print_guard = |
291 | crate::PrintOnPanic(|| format!("terminator {:?}", bb_data.terminator().kind)); | |
292 | ||
29967ef6 XL |
293 | match &bb_data.terminator().kind { |
294 | TerminatorKind::Goto { target } => { | |
295 | if let TerminatorKind::Return = fx.mir[*target].terminator().kind { | |
296 | let mut can_immediately_return = true; | |
297 | for stmt in &fx.mir[*target].statements { | |
298 | if let StatementKind::StorageDead(_) = stmt.kind { | |
299 | } else { | |
300 | // FIXME Can sometimes happen, see rust-lang/rust#70531 | |
301 | can_immediately_return = false; | |
302 | break; | |
303 | } | |
304 | } | |
305 | ||
306 | if can_immediately_return { | |
307 | crate::abi::codegen_return(fx); | |
308 | continue; | |
309 | } | |
310 | } | |
311 | ||
312 | let block = fx.get_block(*target); | |
313 | fx.bcx.ins().jump(block, &[]); | |
314 | } | |
315 | TerminatorKind::Return => { | |
316 | crate::abi::codegen_return(fx); | |
317 | } | |
353b0b11 FG |
318 | TerminatorKind::Assert { cond, expected, msg, target, unwind: _ } => { |
319 | if !fx.tcx.sess.overflow_checks() && msg.is_optional_overflow_check() { | |
320 | let target = fx.get_block(*target); | |
321 | fx.bcx.ins().jump(target, &[]); | |
322 | continue; | |
29967ef6 XL |
323 | } |
324 | let cond = codegen_operand(fx, cond).load_scalar(fx); | |
325 | ||
326 | let target = fx.get_block(*target); | |
327 | let failure = fx.bcx.create_block(); | |
5e7ed085 | 328 | fx.bcx.set_cold_block(failure); |
29967ef6 XL |
329 | |
330 | if *expected { | |
353b0b11 | 331 | fx.bcx.ins().brif(cond, target, &[], failure, &[]); |
29967ef6 | 332 | } else { |
353b0b11 | 333 | fx.bcx.ins().brif(cond, failure, &[], target, &[]); |
29967ef6 | 334 | }; |
29967ef6 XL |
335 | |
336 | fx.bcx.switch_to_block(failure); | |
337 | fx.bcx.ins().nop(); | |
338 | ||
49aad941 | 339 | match &**msg { |
29967ef6 XL |
340 | AssertKind::BoundsCheck { ref len, ref index } => { |
341 | let len = codegen_operand(fx, len).load_scalar(fx); | |
342 | let index = codegen_operand(fx, index).load_scalar(fx); | |
923072b8 | 343 | let location = fx.get_caller_location(source_info).load_scalar(fx); |
29967ef6 XL |
344 | |
345 | codegen_panic_inner( | |
346 | fx, | |
347 | rustc_hir::LangItem::PanicBoundsCheck, | |
348 | &[index, len, location], | |
a2a8927a | 349 | source_info.span, |
29967ef6 XL |
350 | ); |
351 | } | |
353b0b11 FG |
352 | AssertKind::MisalignedPointerDereference { ref required, ref found } => { |
353 | let required = codegen_operand(fx, required).load_scalar(fx); | |
354 | let found = codegen_operand(fx, found).load_scalar(fx); | |
355 | let location = fx.get_caller_location(source_info).load_scalar(fx); | |
356 | ||
357 | codegen_panic_inner( | |
358 | fx, | |
359 | rustc_hir::LangItem::PanicBoundsCheck, | |
360 | &[required, found, location], | |
361 | source_info.span, | |
362 | ); | |
363 | } | |
29967ef6 XL |
364 | _ => { |
365 | let msg_str = msg.description(); | |
923072b8 | 366 | codegen_panic(fx, msg_str, source_info); |
29967ef6 XL |
367 | } |
368 | } | |
369 | } | |
370 | ||
9c376795 FG |
371 | TerminatorKind::SwitchInt { discr, targets } => { |
372 | let discr = codegen_operand(fx, discr); | |
373 | let switch_ty = discr.layout().ty; | |
374 | let discr = discr.load_scalar(fx); | |
29967ef6 | 375 | |
5869c6ff XL |
376 | let use_bool_opt = switch_ty.kind() == fx.tcx.types.bool.kind() |
377 | || (targets.iter().count() == 1 && targets.iter().next().unwrap().0 == 0); | |
378 | if use_bool_opt { | |
29967ef6 XL |
379 | assert_eq!(targets.iter().count(), 1); |
380 | let (then_value, then_block) = targets.iter().next().unwrap(); | |
381 | let then_block = fx.get_block(then_block); | |
382 | let else_block = fx.get_block(targets.otherwise()); | |
383 | let test_zero = match then_value { | |
384 | 0 => true, | |
385 | 1 => false, | |
386 | _ => unreachable!("{:?}", targets), | |
387 | }; | |
388 | ||
29967ef6 XL |
389 | let (discr, is_inverted) = |
390 | crate::optimize::peephole::maybe_unwrap_bool_not(&mut fx.bcx, discr); | |
391 | let test_zero = if is_inverted { !test_zero } else { test_zero }; | |
5869c6ff XL |
392 | if let Some(taken) = crate::optimize::peephole::maybe_known_branch_taken( |
393 | &fx.bcx, discr, test_zero, | |
394 | ) { | |
395 | if taken { | |
396 | fx.bcx.ins().jump(then_block, &[]); | |
397 | } else { | |
398 | fx.bcx.ins().jump(else_block, &[]); | |
399 | } | |
29967ef6 | 400 | } else { |
5869c6ff | 401 | if test_zero { |
353b0b11 | 402 | fx.bcx.ins().brif(discr, else_block, &[], then_block, &[]); |
5869c6ff | 403 | } else { |
353b0b11 | 404 | fx.bcx.ins().brif(discr, then_block, &[], else_block, &[]); |
5869c6ff | 405 | } |
29967ef6 XL |
406 | } |
407 | } else { | |
408 | let mut switch = ::cranelift_frontend::Switch::new(); | |
409 | for (value, block) in targets.iter() { | |
410 | let block = fx.get_block(block); | |
411 | switch.set_entry(value, block); | |
412 | } | |
413 | let otherwise_block = fx.get_block(targets.otherwise()); | |
414 | switch.emit(&mut fx.bcx, discr, otherwise_block); | |
415 | } | |
416 | } | |
417 | TerminatorKind::Call { | |
418 | func, | |
419 | args, | |
420 | destination, | |
923072b8 | 421 | target, |
29967ef6 | 422 | fn_span, |
353b0b11 | 423 | unwind: _, |
fe692bf9 | 424 | call_source: _, |
29967ef6 | 425 | } => { |
9ffffee4 | 426 | fx.tcx.prof.generic_activity("codegen call").run(|| { |
923072b8 FG |
427 | crate::abi::codegen_terminator_call( |
428 | fx, | |
429 | mir::SourceInfo { span: *fn_span, ..source_info }, | |
430 | func, | |
431 | args, | |
432 | *destination, | |
433 | *target, | |
434 | ) | |
29967ef6 XL |
435 | }); |
436 | } | |
437 | TerminatorKind::InlineAsm { | |
438 | template, | |
439 | operands, | |
440 | options, | |
441 | destination, | |
442 | line_spans: _, | |
353b0b11 | 443 | unwind: _, |
29967ef6 | 444 | } => { |
a2a8927a XL |
445 | if options.contains(InlineAsmOptions::MAY_UNWIND) { |
446 | fx.tcx.sess.span_fatal( | |
447 | source_info.span, | |
448 | "cranelift doesn't support unwinding from inline assembly.", | |
449 | ); | |
450 | } | |
451 | ||
29967ef6 XL |
452 | crate::inline_asm::codegen_inline_asm( |
453 | fx, | |
a2a8927a | 454 | source_info.span, |
29967ef6 XL |
455 | template, |
456 | operands, | |
457 | *options, | |
f2b60f7d | 458 | *destination, |
29967ef6 | 459 | ); |
29967ef6 | 460 | } |
353b0b11 | 461 | TerminatorKind::Terminate => { |
9ffffee4 FG |
462 | codegen_panic_cannot_unwind(fx, source_info); |
463 | } | |
464 | TerminatorKind::Resume => { | |
5e7ed085 FG |
465 | // FIXME implement unwinding |
466 | fx.bcx.ins().trap(TrapCode::UnreachableCodeReached); | |
29967ef6 XL |
467 | } |
468 | TerminatorKind::Unreachable => { | |
5e7ed085 | 469 | fx.bcx.ins().trap(TrapCode::UnreachableCodeReached); |
29967ef6 XL |
470 | } |
471 | TerminatorKind::Yield { .. } | |
472 | | TerminatorKind::FalseEdge { .. } | |
473 | | TerminatorKind::FalseUnwind { .. } | |
29967ef6 XL |
474 | | TerminatorKind::GeneratorDrop => { |
475 | bug!("shouldn't exist at codegen {:?}", bb_data.terminator()); | |
476 | } | |
49aad941 | 477 | TerminatorKind::Drop { place, target, unwind: _, replace: _ } => { |
29967ef6 | 478 | let drop_place = codegen_place(fx, *place); |
923072b8 | 479 | crate::abi::codegen_drop(fx, source_info, drop_place); |
29967ef6 XL |
480 | |
481 | let target_block = fx.get_block(*target); | |
482 | fx.bcx.ins().jump(target_block, &[]); | |
483 | } | |
484 | }; | |
485 | } | |
29967ef6 XL |
486 | } |
487 | ||
488 | fn codegen_stmt<'tcx>( | |
6a06907d | 489 | fx: &mut FunctionCx<'_, '_, 'tcx>, |
29967ef6 XL |
490 | #[allow(unused_variables)] cur_block: Block, |
491 | stmt: &Statement<'tcx>, | |
492 | ) { | |
493 | let _print_guard = crate::PrintOnPanic(|| format!("stmt {:?}", stmt)); | |
494 | ||
495 | fx.set_debug_loc(stmt.source_info); | |
496 | ||
29967ef6 XL |
497 | match &stmt.kind { |
498 | StatementKind::StorageLive(..) | StatementKind::StorageDead(..) => {} // Those are not very useful | |
499 | _ => { | |
cdc7bbd5 XL |
500 | if fx.clif_comments.enabled() { |
501 | let inst = fx.bcx.func.layout.last_inst(cur_block).unwrap(); | |
49aad941 FG |
502 | with_no_trimmed_paths!({ |
503 | fx.add_comment(inst, format!("{:?}", stmt)); | |
504 | }); | |
cdc7bbd5 | 505 | } |
29967ef6 XL |
506 | } |
507 | } | |
508 | ||
509 | match &stmt.kind { | |
6a06907d | 510 | StatementKind::SetDiscriminant { place, variant_index } => { |
29967ef6 XL |
511 | let place = codegen_place(fx, **place); |
512 | crate::discriminant::codegen_set_discriminant(fx, place, *variant_index); | |
513 | } | |
514 | StatementKind::Assign(to_place_and_rval) => { | |
515 | let lval = codegen_place(fx, to_place_and_rval.0); | |
516 | let dest_layout = lval.layout(); | |
fc512014 XL |
517 | match to_place_and_rval.1 { |
518 | Rvalue::Use(ref operand) => { | |
29967ef6 XL |
519 | let val = codegen_operand(fx, operand); |
520 | lval.write_cvalue(fx, val); | |
521 | } | |
064997fb FG |
522 | Rvalue::CopyForDeref(place) => { |
523 | let cplace = codegen_place(fx, place); | |
524 | let val = cplace.to_cvalue(fx); | |
525 | lval.write_cvalue(fx, val) | |
526 | } | |
29967ef6 | 527 | Rvalue::Ref(_, _, place) | Rvalue::AddressOf(_, place) => { |
fc512014 | 528 | let place = codegen_place(fx, place); |
29967ef6 XL |
529 | let ref_ = place.place_ref(fx, lval.layout()); |
530 | lval.write_cvalue(fx, ref_); | |
531 | } | |
532 | Rvalue::ThreadLocalRef(def_id) => { | |
fc512014 | 533 | let val = crate::constant::codegen_tls_ref(fx, def_id, lval.layout()); |
29967ef6 XL |
534 | lval.write_cvalue(fx, val); |
535 | } | |
cdc7bbd5 XL |
536 | Rvalue::BinaryOp(bin_op, ref lhs_rhs) => { |
537 | let lhs = codegen_operand(fx, &lhs_rhs.0); | |
538 | let rhs = codegen_operand(fx, &lhs_rhs.1); | |
29967ef6 | 539 | |
fc512014 | 540 | let res = crate::num::codegen_binop(fx, bin_op, lhs, rhs); |
29967ef6 XL |
541 | lval.write_cvalue(fx, res); |
542 | } | |
cdc7bbd5 XL |
543 | Rvalue::CheckedBinaryOp(bin_op, ref lhs_rhs) => { |
544 | let lhs = codegen_operand(fx, &lhs_rhs.0); | |
545 | let rhs = codegen_operand(fx, &lhs_rhs.1); | |
29967ef6 | 546 | |
9ffffee4 | 547 | let res = crate::num::codegen_checked_int_binop(fx, bin_op, lhs, rhs); |
29967ef6 XL |
548 | lval.write_cvalue(fx, res); |
549 | } | |
fc512014 | 550 | Rvalue::UnaryOp(un_op, ref operand) => { |
29967ef6 XL |
551 | let operand = codegen_operand(fx, operand); |
552 | let layout = operand.layout(); | |
553 | let val = operand.load_scalar(fx); | |
554 | let res = match un_op { | |
555 | UnOp::Not => match layout.ty.kind() { | |
556 | ty::Bool => { | |
557 | let res = fx.bcx.ins().icmp_imm(IntCC::Equal, val, 0); | |
9c376795 | 558 | CValue::by_val(res, layout) |
29967ef6 XL |
559 | } |
560 | ty::Uint(_) | ty::Int(_) => { | |
561 | CValue::by_val(fx.bcx.ins().bnot(val), layout) | |
562 | } | |
563 | _ => unreachable!("un op Not for {:?}", layout.ty), | |
564 | }, | |
565 | UnOp::Neg => match layout.ty.kind() { | |
29967ef6 XL |
566 | ty::Int(_) => CValue::by_val(fx.bcx.ins().ineg(val), layout), |
567 | ty::Float(_) => CValue::by_val(fx.bcx.ins().fneg(val), layout), | |
568 | _ => unreachable!("un op Neg for {:?}", layout.ty), | |
569 | }, | |
570 | }; | |
571 | lval.write_cvalue(fx, res); | |
572 | } | |
fc512014 | 573 | Rvalue::Cast( |
fe692bf9 | 574 | CastKind::PointerCoercion(PointerCoercion::ReifyFnPointer), |
fc512014 XL |
575 | ref operand, |
576 | to_ty, | |
577 | ) => { | |
578 | let from_ty = fx.monomorphize(operand.ty(&fx.mir.local_decls, fx.tcx)); | |
29967ef6 XL |
579 | let to_layout = fx.layout_of(fx.monomorphize(to_ty)); |
580 | match *from_ty.kind() { | |
581 | ty::FnDef(def_id, substs) => { | |
582 | let func_ref = fx.get_function_ref( | |
583 | Instance::resolve_for_fn_ptr( | |
584 | fx.tcx, | |
585 | ParamEnv::reveal_all(), | |
586 | def_id, | |
587 | substs, | |
588 | ) | |
589 | .unwrap() | |
590 | .polymorphize(fx.tcx), | |
591 | ); | |
592 | let func_addr = fx.bcx.ins().func_addr(fx.pointer_type, func_ref); | |
593 | lval.write_cvalue(fx, CValue::by_val(func_addr, to_layout)); | |
594 | } | |
595 | _ => bug!("Trying to ReifyFnPointer on non FnDef {:?}", from_ty), | |
596 | } | |
597 | } | |
fc512014 | 598 | Rvalue::Cast( |
fe692bf9 | 599 | CastKind::PointerCoercion(PointerCoercion::UnsafeFnPointer), |
fc512014 XL |
600 | ref operand, |
601 | to_ty, | |
602 | ) | |
603 | | Rvalue::Cast( | |
fe692bf9 | 604 | CastKind::PointerCoercion(PointerCoercion::MutToConstPointer), |
fc512014 XL |
605 | ref operand, |
606 | to_ty, | |
607 | ) | |
608 | | Rvalue::Cast( | |
fe692bf9 | 609 | CastKind::PointerCoercion(PointerCoercion::ArrayToPointer), |
fc512014 XL |
610 | ref operand, |
611 | to_ty, | |
612 | ) => { | |
29967ef6 XL |
613 | let to_layout = fx.layout_of(fx.monomorphize(to_ty)); |
614 | let operand = codegen_operand(fx, operand); | |
615 | lval.write_cvalue(fx, operand.cast_pointer_to(to_layout)); | |
616 | } | |
923072b8 | 617 | Rvalue::Cast( |
2b03887a FG |
618 | CastKind::IntToInt |
619 | | CastKind::FloatToFloat | |
620 | | CastKind::FloatToInt | |
621 | | CastKind::IntToFloat | |
622 | | CastKind::FnPtrToPtr | |
623 | | CastKind::PtrToPtr | |
923072b8 FG |
624 | | CastKind::PointerExposeAddress |
625 | | CastKind::PointerFromExposedAddress, | |
626 | ref operand, | |
627 | to_ty, | |
628 | ) => { | |
29967ef6 XL |
629 | let operand = codegen_operand(fx, operand); |
630 | let from_ty = operand.layout().ty; | |
631 | let to_ty = fx.monomorphize(to_ty); | |
632 | ||
6a06907d | 633 | fn is_fat_ptr<'tcx>(fx: &FunctionCx<'_, '_, 'tcx>, ty: Ty<'tcx>) -> bool { |
49aad941 FG |
634 | ty.builtin_deref(true).is_some_and( |
635 | |ty::TypeAndMut { ty: pointee_ty, mutbl: _ }| { | |
6a06907d | 636 | has_ptr_meta(fx.tcx, pointee_ty) |
49aad941 FG |
637 | }, |
638 | ) | |
29967ef6 XL |
639 | } |
640 | ||
641 | if is_fat_ptr(fx, from_ty) { | |
642 | if is_fat_ptr(fx, to_ty) { | |
643 | // fat-ptr -> fat-ptr | |
644 | lval.write_cvalue(fx, operand.cast_pointer_to(dest_layout)); | |
645 | } else { | |
646 | // fat-ptr -> thin-ptr | |
647 | let (ptr, _extra) = operand.load_scalar_pair(fx); | |
648 | lval.write_cvalue(fx, CValue::by_val(ptr, dest_layout)) | |
649 | } | |
29967ef6 XL |
650 | } else { |
651 | let to_clif_ty = fx.clif_type(to_ty).unwrap(); | |
652 | let from = operand.load_scalar(fx); | |
653 | ||
654 | let res = clif_int_or_float_cast( | |
655 | fx, | |
656 | from, | |
657 | type_sign(from_ty), | |
658 | to_clif_ty, | |
659 | type_sign(to_ty), | |
660 | ); | |
661 | lval.write_cvalue(fx, CValue::by_val(res, dest_layout)); | |
662 | } | |
663 | } | |
664 | Rvalue::Cast( | |
fe692bf9 | 665 | CastKind::PointerCoercion(PointerCoercion::ClosureFnPointer(_)), |
fc512014 | 666 | ref operand, |
29967ef6 XL |
667 | _to_ty, |
668 | ) => { | |
669 | let operand = codegen_operand(fx, operand); | |
670 | match *operand.layout().ty.kind() { | |
671 | ty::Closure(def_id, substs) => { | |
672 | let instance = Instance::resolve_closure( | |
673 | fx.tcx, | |
674 | def_id, | |
675 | substs, | |
676 | ty::ClosureKind::FnOnce, | |
677 | ) | |
064997fb | 678 | .expect("failed to normalize and resolve closure during codegen") |
29967ef6 XL |
679 | .polymorphize(fx.tcx); |
680 | let func_ref = fx.get_function_ref(instance); | |
681 | let func_addr = fx.bcx.ins().func_addr(fx.pointer_type, func_ref); | |
682 | lval.write_cvalue(fx, CValue::by_val(func_addr, lval.layout())); | |
683 | } | |
684 | _ => bug!("{} cannot be cast to a fn ptr", operand.layout().ty), | |
685 | } | |
686 | } | |
fe692bf9 FG |
687 | Rvalue::Cast( |
688 | CastKind::PointerCoercion(PointerCoercion::Unsize), | |
689 | ref operand, | |
690 | _to_ty, | |
691 | ) => { | |
29967ef6 | 692 | let operand = codegen_operand(fx, operand); |
49aad941 | 693 | crate::unsize::coerce_unsized_into(fx, operand, lval); |
29967ef6 | 694 | } |
2b03887a FG |
695 | Rvalue::Cast(CastKind::DynStar, ref operand, _) => { |
696 | let operand = codegen_operand(fx, operand); | |
49aad941 | 697 | crate::unsize::coerce_dyn_star(fx, operand, lval); |
f2b60f7d | 698 | } |
353b0b11 FG |
699 | Rvalue::Cast(CastKind::Transmute, ref operand, _to_ty) => { |
700 | let operand = codegen_operand(fx, operand); | |
701 | lval.write_cvalue_transmute(fx, operand); | |
702 | } | |
29967ef6 | 703 | Rvalue::Discriminant(place) => { |
fc512014 | 704 | let place = codegen_place(fx, place); |
29967ef6 | 705 | let value = place.to_cvalue(fx); |
f2b60f7d | 706 | crate::discriminant::codegen_get_discriminant(fx, lval, value, dest_layout); |
29967ef6 | 707 | } |
fc512014 | 708 | Rvalue::Repeat(ref operand, times) => { |
29967ef6 XL |
709 | let operand = codegen_operand(fx, operand); |
710 | let times = fx | |
711 | .monomorphize(times) | |
712 | .eval(fx.tcx, ParamEnv::reveal_all()) | |
29967ef6 XL |
713 | .try_to_bits(fx.tcx.data_layout.pointer_size) |
714 | .unwrap(); | |
cdc7bbd5 XL |
715 | if operand.layout().size.bytes() == 0 { |
716 | // Do nothing for ZST's | |
717 | } else if fx.clif_type(operand.layout().ty) == Some(types::I8) { | |
29967ef6 XL |
718 | let times = fx.bcx.ins().iconst(fx.pointer_type, times as i64); |
719 | // FIXME use emit_small_memset where possible | |
720 | let addr = lval.to_ptr().get_addr(fx); | |
721 | let val = operand.load_scalar(fx); | |
a2a8927a | 722 | fx.bcx.call_memset(fx.target_config, addr, val, times); |
29967ef6 XL |
723 | } else { |
724 | let loop_block = fx.bcx.create_block(); | |
725 | let loop_block2 = fx.bcx.create_block(); | |
726 | let done_block = fx.bcx.create_block(); | |
727 | let index = fx.bcx.append_block_param(loop_block, fx.pointer_type); | |
728 | let zero = fx.bcx.ins().iconst(fx.pointer_type, 0); | |
729 | fx.bcx.ins().jump(loop_block, &[zero]); | |
730 | ||
731 | fx.bcx.switch_to_block(loop_block); | |
732 | let done = fx.bcx.ins().icmp_imm(IntCC::Equal, index, times as i64); | |
353b0b11 | 733 | fx.bcx.ins().brif(done, done_block, &[], loop_block2, &[]); |
29967ef6 XL |
734 | |
735 | fx.bcx.switch_to_block(loop_block2); | |
736 | let to = lval.place_index(fx, index); | |
737 | to.write_cvalue(fx, operand); | |
738 | let index = fx.bcx.ins().iadd_imm(index, 1); | |
739 | fx.bcx.ins().jump(loop_block, &[index]); | |
740 | ||
741 | fx.bcx.switch_to_block(done_block); | |
742 | fx.bcx.ins().nop(); | |
743 | } | |
744 | } | |
745 | Rvalue::Len(place) => { | |
fc512014 | 746 | let place = codegen_place(fx, place); |
29967ef6 XL |
747 | let usize_layout = fx.layout_of(fx.tcx.types.usize); |
748 | let len = codegen_array_len(fx, place); | |
749 | lval.write_cvalue(fx, CValue::by_val(len, usize_layout)); | |
750 | } | |
c295e0f8 XL |
751 | Rvalue::ShallowInitBox(ref operand, content_ty) => { |
752 | let content_ty = fx.monomorphize(content_ty); | |
fe692bf9 | 753 | let box_layout = fx.layout_of(Ty::new_box(fx.tcx, content_ty)); |
c295e0f8 XL |
754 | let operand = codegen_operand(fx, operand); |
755 | let operand = operand.load_scalar(fx); | |
756 | lval.write_cvalue(fx, CValue::by_val(operand, box_layout)); | |
757 | } | |
49aad941 | 758 | Rvalue::NullaryOp(ref null_op, ty) => { |
2b03887a | 759 | assert!(lval.layout().ty.is_sized(fx.tcx, ParamEnv::reveal_all())); |
c295e0f8 XL |
760 | let layout = fx.layout_of(fx.monomorphize(ty)); |
761 | let val = match null_op { | |
762 | NullOp::SizeOf => layout.size.bytes(), | |
763 | NullOp::AlignOf => layout.align.abi.bytes(), | |
49aad941 FG |
764 | NullOp::OffsetOf(fields) => { |
765 | layout.offset_of_subfield(fx, fields.iter().map(|f| f.index())).bytes() | |
766 | } | |
c295e0f8 | 767 | }; |
49aad941 FG |
768 | let val = CValue::by_val( |
769 | fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(val).unwrap()), | |
770 | fx.layout_of(fx.tcx.types.usize), | |
771 | ); | |
29967ef6 XL |
772 | lval.write_cvalue(fx, val); |
773 | } | |
9ffffee4 FG |
774 | Rvalue::Aggregate(ref kind, ref operands) => { |
775 | let (variant_index, variant_dest, active_field_index) = match **kind { | |
776 | mir::AggregateKind::Adt(_, variant_index, _, _, active_field_index) => { | |
777 | let variant_dest = lval.downcast_variant(fx, variant_index); | |
778 | (variant_index, variant_dest, active_field_index) | |
29967ef6 | 779 | } |
353b0b11 | 780 | _ => (FIRST_VARIANT, lval, None), |
9ffffee4 FG |
781 | }; |
782 | if active_field_index.is_some() { | |
783 | assert_eq!(operands.len(), 1); | |
29967ef6 | 784 | } |
353b0b11 | 785 | for (i, operand) in operands.iter_enumerated() { |
9ffffee4 FG |
786 | let operand = codegen_operand(fx, operand); |
787 | let field_index = active_field_index.unwrap_or(i); | |
788 | let to = if let mir::AggregateKind::Array(_) = **kind { | |
353b0b11 FG |
789 | let array_index = i64::from(field_index.as_u32()); |
790 | let index = fx.bcx.ins().iconst(fx.pointer_type, array_index); | |
9ffffee4 FG |
791 | variant_dest.place_index(fx, index) |
792 | } else { | |
353b0b11 | 793 | variant_dest.place_field(fx, field_index) |
9ffffee4 FG |
794 | }; |
795 | to.write_cvalue(fx, operand); | |
796 | } | |
797 | crate::discriminant::codegen_set_discriminant(fx, lval, variant_index); | |
798 | } | |
29967ef6 XL |
799 | } |
800 | } | |
801 | StatementKind::StorageLive(_) | |
802 | | StatementKind::StorageDead(_) | |
04454e1e | 803 | | StatementKind::Deinit(_) |
9ffffee4 | 804 | | StatementKind::ConstEvalCounter |
29967ef6 XL |
805 | | StatementKind::Nop |
806 | | StatementKind::FakeRead(..) | |
807 | | StatementKind::Retag { .. } | |
353b0b11 | 808 | | StatementKind::PlaceMention(..) |
29967ef6 XL |
809 | | StatementKind::AscribeUserType(..) => {} |
810 | ||
29967ef6 | 811 | StatementKind::Coverage { .. } => fx.tcx.sess.fatal("-Zcoverage is unimplemented"), |
f2b60f7d FG |
812 | StatementKind::Intrinsic(ref intrinsic) => match &**intrinsic { |
813 | // We ignore `assume` intrinsics, they are only useful for optimizations | |
814 | NonDivergingIntrinsic::Assume(_) => {} | |
815 | NonDivergingIntrinsic::CopyNonOverlapping(mir::CopyNonOverlapping { | |
816 | src, | |
817 | dst, | |
818 | count, | |
819 | }) => { | |
820 | let dst = codegen_operand(fx, dst); | |
821 | let pointee = dst | |
822 | .layout() | |
823 | .pointee_info_at(fx, rustc_target::abi::Size::ZERO) | |
824 | .expect("Expected pointer"); | |
825 | let dst = dst.load_scalar(fx); | |
826 | let src = codegen_operand(fx, src).load_scalar(fx); | |
827 | let count = codegen_operand(fx, count).load_scalar(fx); | |
828 | let elem_size: u64 = pointee.size.bytes(); | |
829 | let bytes = if elem_size != 1 { | |
830 | fx.bcx.ins().imul_imm(count, elem_size as i64) | |
831 | } else { | |
832 | count | |
833 | }; | |
834 | fx.bcx.call_memcpy(fx.target_config, dst, src, bytes); | |
835 | } | |
836 | }, | |
29967ef6 XL |
837 | } |
838 | } | |
839 | ||
6a06907d | 840 | fn codegen_array_len<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, place: CPlace<'tcx>) -> Value { |
29967ef6 XL |
841 | match *place.layout().ty.kind() { |
842 | ty::Array(_elem_ty, len) => { | |
9ffffee4 | 843 | let len = fx.monomorphize(len).eval_target_usize(fx.tcx, ParamEnv::reveal_all()) as i64; |
29967ef6 XL |
844 | fx.bcx.ins().iconst(fx.pointer_type, len) |
845 | } | |
49aad941 | 846 | ty::Slice(_elem_ty) => place.to_ptr_unsized().1, |
29967ef6 XL |
847 | _ => bug!("Rvalue::Len({:?})", place), |
848 | } | |
849 | } | |
850 | ||
851 | pub(crate) fn codegen_place<'tcx>( | |
6a06907d | 852 | fx: &mut FunctionCx<'_, '_, 'tcx>, |
29967ef6 XL |
853 | place: Place<'tcx>, |
854 | ) -> CPlace<'tcx> { | |
855 | let mut cplace = fx.get_local_place(place.local); | |
856 | ||
857 | for elem in place.projection { | |
858 | match elem { | |
859 | PlaceElem::Deref => { | |
064997fb | 860 | cplace = cplace.place_deref(fx); |
29967ef6 | 861 | } |
2b03887a | 862 | PlaceElem::OpaqueCast(ty) => cplace = cplace.place_opaque_cast(fx, ty), |
29967ef6 XL |
863 | PlaceElem::Field(field, _ty) => { |
864 | cplace = cplace.place_field(fx, field); | |
865 | } | |
866 | PlaceElem::Index(local) => { | |
867 | let index = fx.get_local_place(local).to_cvalue(fx).load_scalar(fx); | |
868 | cplace = cplace.place_index(fx, index); | |
869 | } | |
6a06907d | 870 | PlaceElem::ConstantIndex { offset, min_length: _, from_end } => { |
29967ef6 XL |
871 | let offset: u64 = offset; |
872 | let index = if !from_end { | |
873 | fx.bcx.ins().iconst(fx.pointer_type, offset as i64) | |
874 | } else { | |
875 | let len = codegen_array_len(fx, cplace); | |
876 | fx.bcx.ins().iadd_imm(len, -(offset as i64)) | |
877 | }; | |
878 | cplace = cplace.place_index(fx, index); | |
879 | } | |
880 | PlaceElem::Subslice { from, to, from_end } => { | |
881 | // These indices are generated by slice patterns. | |
882 | // slice[from:-to] in Python terms. | |
883 | ||
884 | let from: u64 = from; | |
885 | let to: u64 = to; | |
886 | ||
887 | match cplace.layout().ty.kind() { | |
888 | ty::Array(elem_ty, _len) => { | |
889 | assert!(!from_end, "array subslices are never `from_end`"); | |
5099ac24 | 890 | let elem_layout = fx.layout_of(*elem_ty); |
29967ef6 XL |
891 | let ptr = cplace.to_ptr(); |
892 | cplace = CPlace::for_ptr( | |
893 | ptr.offset_i64(fx, elem_layout.size.bytes() as i64 * (from as i64)), | |
fe692bf9 | 894 | fx.layout_of(Ty::new_array(fx.tcx, *elem_ty, to - from)), |
29967ef6 XL |
895 | ); |
896 | } | |
897 | ty::Slice(elem_ty) => { | |
898 | assert!(from_end, "slice subslices should be `from_end`"); | |
5099ac24 | 899 | let elem_layout = fx.layout_of(*elem_ty); |
49aad941 | 900 | let (ptr, len) = cplace.to_ptr_unsized(); |
29967ef6 XL |
901 | cplace = CPlace::for_ptr_with_extra( |
902 | ptr.offset_i64(fx, elem_layout.size.bytes() as i64 * (from as i64)), | |
903 | fx.bcx.ins().iadd_imm(len, -(from as i64 + to as i64)), | |
904 | cplace.layout(), | |
905 | ); | |
906 | } | |
907 | _ => unreachable!(), | |
908 | } | |
909 | } | |
910 | PlaceElem::Downcast(_adt_def, variant) => { | |
911 | cplace = cplace.downcast_variant(fx, variant); | |
912 | } | |
913 | } | |
914 | } | |
915 | ||
916 | cplace | |
917 | } | |
918 | ||
919 | pub(crate) fn codegen_operand<'tcx>( | |
6a06907d | 920 | fx: &mut FunctionCx<'_, '_, 'tcx>, |
29967ef6 XL |
921 | operand: &Operand<'tcx>, |
922 | ) -> CValue<'tcx> { | |
923 | match operand { | |
924 | Operand::Move(place) | Operand::Copy(place) => { | |
925 | let cplace = codegen_place(fx, *place); | |
926 | cplace.to_cvalue(fx) | |
927 | } | |
2b03887a | 928 | Operand::Constant(const_) => crate::constant::codegen_constant_operand(fx, const_), |
29967ef6 XL |
929 | } |
930 | } | |
931 | ||
923072b8 FG |
932 | pub(crate) fn codegen_panic<'tcx>( |
933 | fx: &mut FunctionCx<'_, '_, 'tcx>, | |
934 | msg_str: &str, | |
935 | source_info: mir::SourceInfo, | |
936 | ) { | |
937 | let location = fx.get_caller_location(source_info).load_scalar(fx); | |
29967ef6 | 938 | |
17df50a5 | 939 | let msg_ptr = fx.anonymous_str(msg_str); |
6a06907d | 940 | let msg_len = fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(msg_str.len()).unwrap()); |
29967ef6 XL |
941 | let args = [msg_ptr, msg_len, location]; |
942 | ||
923072b8 | 943 | codegen_panic_inner(fx, rustc_hir::LangItem::Panic, &args, source_info.span); |
29967ef6 XL |
944 | } |
945 | ||
9ffffee4 FG |
946 | pub(crate) fn codegen_panic_nounwind<'tcx>( |
947 | fx: &mut FunctionCx<'_, '_, 'tcx>, | |
948 | msg_str: &str, | |
949 | source_info: mir::SourceInfo, | |
950 | ) { | |
951 | let msg_ptr = fx.anonymous_str(msg_str); | |
952 | let msg_len = fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(msg_str.len()).unwrap()); | |
953 | let args = [msg_ptr, msg_len]; | |
954 | ||
955 | codegen_panic_inner(fx, rustc_hir::LangItem::PanicNounwind, &args, source_info.span); | |
956 | } | |
957 | ||
958 | pub(crate) fn codegen_panic_cannot_unwind<'tcx>( | |
959 | fx: &mut FunctionCx<'_, '_, 'tcx>, | |
960 | source_info: mir::SourceInfo, | |
961 | ) { | |
962 | let args = []; | |
963 | ||
964 | codegen_panic_inner(fx, rustc_hir::LangItem::PanicCannotUnwind, &args, source_info.span); | |
965 | } | |
966 | ||
967 | fn codegen_panic_inner<'tcx>( | |
6a06907d | 968 | fx: &mut FunctionCx<'_, '_, 'tcx>, |
29967ef6 XL |
969 | lang_item: rustc_hir::LangItem, |
970 | args: &[Value], | |
971 | span: Span, | |
972 | ) { | |
49aad941 | 973 | let def_id = fx.tcx.require_lang_item(lang_item, Some(span)); |
29967ef6 XL |
974 | |
975 | let instance = Instance::mono(fx.tcx, def_id).polymorphize(fx.tcx); | |
976 | let symbol_name = fx.tcx.symbol_name(instance).name; | |
977 | ||
978 | fx.lib_call( | |
353b0b11 | 979 | symbol_name, |
9ffffee4 | 980 | args.iter().map(|&arg| AbiParam::new(fx.bcx.func.dfg.value_type(arg))).collect(), |
29967ef6 XL |
981 | vec![], |
982 | args, | |
983 | ); | |
984 | ||
5e7ed085 | 985 | fx.bcx.ins().trap(TrapCode::UnreachableCodeReached); |
29967ef6 | 986 | } |