]>
Commit | Line | Data |
---|---|---|
29967ef6 XL |
1 | //! Handling of everything related to the calling convention. Also fills `fx.local_map`. |
2 | ||
29967ef6 XL |
3 | mod comments; |
4 | mod pass_mode; | |
5 | mod returning; | |
6 | ||
353b0b11 FG |
7 | use std::borrow::Cow; |
8 | ||
064997fb | 9 | use cranelift_module::ModuleError; |
29967ef6 | 10 | use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; |
c295e0f8 | 11 | use rustc_middle::ty::layout::FnAbiOf; |
9ffffee4 | 12 | use rustc_session::Session; |
5869c6ff | 13 | use rustc_target::abi::call::{Conv, FnAbi}; |
29967ef6 XL |
14 | use rustc_target::spec::abi::Abi; |
15 | ||
94222f64 | 16 | use cranelift_codegen::ir::{AbiParam, SigRef}; |
29967ef6 XL |
17 | |
18 | use self::pass_mode::*; | |
19 | use crate::prelude::*; | |
20 | ||
94222f64 | 21 | pub(crate) use self::returning::codegen_return; |
29967ef6 | 22 | |
5869c6ff | 23 | fn clif_sig_from_fn_abi<'tcx>( |
29967ef6 | 24 | tcx: TyCtxt<'tcx>, |
a2a8927a | 25 | default_call_conv: CallConv, |
5869c6ff | 26 | fn_abi: &FnAbi<'tcx, Ty<'tcx>>, |
29967ef6 | 27 | ) -> Signature { |
9ffffee4 | 28 | let call_conv = conv_to_call_conv(tcx.sess, fn_abi.conv, default_call_conv); |
487cf647 | 29 | |
353b0b11 | 30 | let inputs = fn_abi.args.iter().flat_map(|arg_abi| arg_abi.get_abi_param(tcx).into_iter()); |
487cf647 FG |
31 | |
32 | let (return_ptr, returns) = fn_abi.ret.get_abi_return(tcx); | |
33 | // Sometimes the first param is an pointer to the place where the return value needs to be stored. | |
34 | let params: Vec<_> = return_ptr.into_iter().chain(inputs).collect(); | |
35 | ||
36 | Signature { params, returns, call_conv } | |
37 | } | |
38 | ||
9ffffee4 | 39 | pub(crate) fn conv_to_call_conv(sess: &Session, c: Conv, default_call_conv: CallConv) -> CallConv { |
487cf647 | 40 | match c { |
a2a8927a | 41 | Conv::Rust | Conv::C => default_call_conv, |
923072b8 | 42 | Conv::RustCold => CallConv::Cold, |
5869c6ff XL |
43 | Conv::X86_64SysV => CallConv::SystemV, |
44 | Conv::X86_64Win64 => CallConv::WindowsFastcall, | |
9ffffee4 FG |
45 | |
46 | // Should already get a back compat warning | |
47 | Conv::X86Fastcall | Conv::X86Stdcall | Conv::X86ThisCall | Conv::X86VectorCall => { | |
48 | default_call_conv | |
49 | } | |
50 | ||
51 | Conv::X86Intr => sess.fatal("x86-interrupt call conv not yet implemented"), | |
52 | ||
53 | Conv::ArmAapcs => sess.fatal("aapcs call conv not yet implemented"), | |
54 | Conv::CCmseNonSecureCall => { | |
55 | sess.fatal("C-cmse-nonsecure-call call conv is not yet implemented"); | |
56 | } | |
57 | ||
58 | Conv::Msp430Intr | |
5869c6ff | 59 | | Conv::PtxKernel |
5869c6ff XL |
60 | | Conv::AmdGpuKernel |
61 | | Conv::AvrInterrupt | |
9ffffee4 FG |
62 | | Conv::AvrNonBlockingInterrupt => { |
63 | unreachable!("tried to use {c:?} call conv which only exists on an unsupported target"); | |
64 | } | |
487cf647 | 65 | } |
29967ef6 XL |
66 | } |
67 | ||
5869c6ff | 68 | pub(crate) fn get_function_sig<'tcx>( |
29967ef6 | 69 | tcx: TyCtxt<'tcx>, |
9c376795 | 70 | default_call_conv: CallConv, |
29967ef6 | 71 | inst: Instance<'tcx>, |
5869c6ff | 72 | ) -> Signature { |
29967ef6 | 73 | assert!(!inst.substs.needs_infer()); |
c295e0f8 XL |
74 | clif_sig_from_fn_abi( |
75 | tcx, | |
9c376795 | 76 | default_call_conv, |
c295e0f8 XL |
77 | &RevealAllLayoutCx(tcx).fn_abi_of_instance(inst, ty::List::empty()), |
78 | ) | |
29967ef6 XL |
79 | } |
80 | ||
81 | /// Instance must be monomorphized | |
82 | pub(crate) fn import_function<'tcx>( | |
83 | tcx: TyCtxt<'tcx>, | |
6a06907d | 84 | module: &mut dyn Module, |
29967ef6 XL |
85 | inst: Instance<'tcx>, |
86 | ) -> FuncId { | |
17df50a5 | 87 | let name = tcx.symbol_name(inst).name; |
9c376795 | 88 | let sig = get_function_sig(tcx, module.target_config().default_call_conv, inst); |
064997fb FG |
89 | match module.declare_function(name, Linkage::Import, &sig) { |
90 | Ok(func_id) => func_id, | |
91 | Err(ModuleError::IncompatibleDeclaration(_)) => tcx.sess.fatal(&format!( | |
92 | "attempt to declare `{name}` as function, but it was already declared as static" | |
93 | )), | |
94 | Err(ModuleError::IncompatibleSignature(_, prev_sig, new_sig)) => tcx.sess.fatal(&format!( | |
95 | "attempt to declare `{name}` with signature {new_sig:?}, \ | |
96 | but it was already declared with signature {prev_sig:?}" | |
97 | )), | |
98 | Err(err) => Err::<_, _>(err).unwrap(), | |
99 | } | |
29967ef6 XL |
100 | } |
101 | ||
6a06907d | 102 | impl<'tcx> FunctionCx<'_, '_, 'tcx> { |
29967ef6 XL |
103 | /// Instance must be monomorphized |
104 | pub(crate) fn get_function_ref(&mut self, inst: Instance<'tcx>) -> FuncRef { | |
17df50a5 XL |
105 | let func_id = import_function(self.tcx, self.module, inst); |
106 | let func_ref = self.module.declare_func_in_func(func_id, &mut self.bcx.func); | |
29967ef6 | 107 | |
cdc7bbd5 XL |
108 | if self.clif_comments.enabled() { |
109 | self.add_comment(func_ref, format!("{:?}", inst)); | |
110 | } | |
29967ef6 XL |
111 | |
112 | func_ref | |
113 | } | |
114 | ||
115 | pub(crate) fn lib_call( | |
116 | &mut self, | |
117 | name: &str, | |
5869c6ff XL |
118 | params: Vec<AbiParam>, |
119 | returns: Vec<AbiParam>, | |
29967ef6 | 120 | args: &[Value], |
353b0b11 FG |
121 | ) -> Cow<'_, [Value]> { |
122 | if self.tcx.sess.target.is_like_windows { | |
123 | let (mut params, mut args): (Vec<_>, Vec<_>) = | |
124 | params | |
125 | .into_iter() | |
126 | .zip(args) | |
127 | .map(|(param, &arg)| { | |
128 | if param.value_type == types::I128 { | |
129 | let arg_ptr = Pointer::stack_slot(self.bcx.create_sized_stack_slot( | |
130 | StackSlotData { kind: StackSlotKind::ExplicitSlot, size: 16 }, | |
131 | )); | |
132 | arg_ptr.store(self, arg, MemFlags::trusted()); | |
133 | (AbiParam::new(self.pointer_type), arg_ptr.get_addr(self)) | |
134 | } else { | |
135 | (param, arg) | |
136 | } | |
137 | }) | |
138 | .unzip(); | |
139 | ||
140 | let indirect_ret_val = returns.len() == 1 && returns[0].value_type == types::I128; | |
141 | ||
142 | if indirect_ret_val { | |
143 | params.insert(0, AbiParam::new(self.pointer_type)); | |
144 | let ret_ptr = | |
145 | Pointer::stack_slot(self.bcx.create_sized_stack_slot(StackSlotData { | |
146 | kind: StackSlotKind::ExplicitSlot, | |
147 | size: 16, | |
148 | })); | |
149 | args.insert(0, ret_ptr.get_addr(self)); | |
150 | self.lib_call_unadjusted(name, params, vec![], &args); | |
151 | return Cow::Owned(vec![ret_ptr.load(self, types::I128, MemFlags::trusted())]); | |
152 | } else { | |
153 | return self.lib_call_unadjusted(name, params, returns, &args); | |
154 | } | |
155 | } | |
156 | ||
157 | self.lib_call_unadjusted(name, params, returns, args) | |
158 | } | |
159 | ||
160 | pub(crate) fn lib_call_unadjusted( | |
161 | &mut self, | |
162 | name: &str, | |
163 | params: Vec<AbiParam>, | |
164 | returns: Vec<AbiParam>, | |
165 | args: &[Value], | |
166 | ) -> Cow<'_, [Value]> { | |
a2a8927a | 167 | let sig = Signature { params, returns, call_conv: self.target_config.default_call_conv }; |
17df50a5 XL |
168 | let func_id = self.module.declare_function(name, Linkage::Import, &sig).unwrap(); |
169 | let func_ref = self.module.declare_func_in_func(func_id, &mut self.bcx.func); | |
5e7ed085 FG |
170 | if self.clif_comments.enabled() { |
171 | self.add_comment(func_ref, format!("{:?}", name)); | |
172 | } | |
29967ef6 | 173 | let call_inst = self.bcx.ins().call(func_ref, args); |
cdc7bbd5 | 174 | if self.clif_comments.enabled() { |
353b0b11 | 175 | self.add_comment(call_inst, format!("lib_call {}", name)); |
29967ef6 XL |
176 | } |
177 | let results = self.bcx.inst_results(call_inst); | |
178 | assert!(results.len() <= 2, "{}", results.len()); | |
353b0b11 | 179 | Cow::Borrowed(results) |
29967ef6 XL |
180 | } |
181 | } | |
182 | ||
183 | /// Make a [`CPlace`] capable of holding value of the specified type. | |
184 | fn make_local_place<'tcx>( | |
6a06907d | 185 | fx: &mut FunctionCx<'_, '_, 'tcx>, |
29967ef6 XL |
186 | local: Local, |
187 | layout: TyAndLayout<'tcx>, | |
188 | is_ssa: bool, | |
189 | ) -> CPlace<'tcx> { | |
9ffffee4 FG |
190 | if layout.is_unsized() { |
191 | fx.tcx.sess.span_fatal( | |
192 | fx.mir.local_decls[local].source_info.span, | |
193 | "unsized locals are not yet supported", | |
194 | ); | |
195 | } | |
29967ef6 XL |
196 | let place = if is_ssa { |
197 | if let rustc_target::abi::Abi::ScalarPair(_, _) = layout.abi { | |
198 | CPlace::new_var_pair(fx, local, layout) | |
199 | } else { | |
200 | CPlace::new_var(fx, local, layout) | |
201 | } | |
202 | } else { | |
203 | CPlace::new_stack_slot(fx, layout) | |
204 | }; | |
205 | ||
29967ef6 XL |
206 | self::comments::add_local_place_comments(fx, place, local); |
207 | ||
208 | place | |
209 | } | |
210 | ||
6a06907d | 211 | pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_block: Block) { |
5869c6ff XL |
212 | fx.bcx.append_block_params_for_function_params(start_block); |
213 | ||
214 | fx.bcx.switch_to_block(start_block); | |
215 | fx.bcx.ins().nop(); | |
216 | ||
29967ef6 XL |
217 | let ssa_analyzed = crate::analyze::analyze(fx); |
218 | ||
29967ef6 XL |
219 | self::comments::add_args_header_comment(fx); |
220 | ||
6a06907d | 221 | let mut block_params_iter = fx.bcx.func.dfg.block_params(start_block).to_vec().into_iter(); |
5869c6ff XL |
222 | let ret_place = |
223 | self::returning::codegen_return_param(fx, &ssa_analyzed, &mut block_params_iter); | |
29967ef6 XL |
224 | assert_eq!(fx.local_map.push(ret_place), RETURN_PLACE); |
225 | ||
226 | // None means pass_mode == NoPass | |
227 | enum ArgKind<'tcx> { | |
228 | Normal(Option<CValue<'tcx>>), | |
229 | Spread(Vec<Option<CValue<'tcx>>>), | |
230 | } | |
231 | ||
5869c6ff | 232 | let fn_abi = fx.fn_abi.take().unwrap(); |
064997fb FG |
233 | |
234 | // FIXME implement variadics in cranelift | |
235 | if fn_abi.c_variadic { | |
236 | fx.tcx.sess.span_fatal( | |
237 | fx.mir.span, | |
238 | "Defining variadic functions is not yet supported by Cranelift", | |
239 | ); | |
240 | } | |
241 | ||
5869c6ff XL |
242 | let mut arg_abis_iter = fn_abi.args.iter(); |
243 | ||
29967ef6 XL |
244 | let func_params = fx |
245 | .mir | |
246 | .args_iter() | |
247 | .map(|local| { | |
fc512014 | 248 | let arg_ty = fx.monomorphize(fx.mir.local_decls[local].ty); |
29967ef6 XL |
249 | |
250 | // Adapted from https://github.com/rust-lang/rust/blob/145155dc96757002c7b2e9de8489416e2fdbbd57/src/librustc_codegen_llvm/mir/mod.rs#L442-L482 | |
251 | if Some(local) == fx.mir.spread_arg { | |
252 | // This argument (e.g. the last argument in the "rust-call" ABI) | |
253 | // is a tuple that was spread at the ABI level and now we have | |
254 | // to reconstruct it into a tuple local variable, from multiple | |
255 | // individual function arguments. | |
256 | ||
257 | let tupled_arg_tys = match arg_ty.kind() { | |
258 | ty::Tuple(ref tys) => tys, | |
259 | _ => bug!("spread argument isn't a tuple?! but {:?}", arg_ty), | |
260 | }; | |
261 | ||
262 | let mut params = Vec::new(); | |
5e7ed085 | 263 | for (i, _arg_ty) in tupled_arg_tys.iter().enumerate() { |
5869c6ff XL |
264 | let arg_abi = arg_abis_iter.next().unwrap(); |
265 | let param = | |
266 | cvalue_for_param(fx, Some(local), Some(i), arg_abi, &mut block_params_iter); | |
29967ef6 XL |
267 | params.push(param); |
268 | } | |
269 | ||
270 | (local, ArgKind::Spread(params), arg_ty) | |
271 | } else { | |
5869c6ff XL |
272 | let arg_abi = arg_abis_iter.next().unwrap(); |
273 | let param = | |
274 | cvalue_for_param(fx, Some(local), None, arg_abi, &mut block_params_iter); | |
29967ef6 XL |
275 | (local, ArgKind::Normal(param), arg_ty) |
276 | } | |
277 | }) | |
278 | .collect::<Vec<(Local, ArgKind<'tcx>, Ty<'tcx>)>>(); | |
279 | ||
280 | assert!(fx.caller_location.is_none()); | |
281 | if fx.instance.def.requires_caller_location(fx.tcx) { | |
282 | // Store caller location for `#[track_caller]`. | |
5869c6ff XL |
283 | let arg_abi = arg_abis_iter.next().unwrap(); |
284 | fx.caller_location = | |
285 | Some(cvalue_for_param(fx, None, None, arg_abi, &mut block_params_iter).unwrap()); | |
29967ef6 XL |
286 | } |
287 | ||
5869c6ff XL |
288 | assert!(arg_abis_iter.next().is_none(), "ArgAbi left behind"); |
289 | fx.fn_abi = Some(fn_abi); | |
290 | assert!(block_params_iter.next().is_none(), "arg_value left behind"); | |
29967ef6 | 291 | |
29967ef6 XL |
292 | self::comments::add_locals_header_comment(fx); |
293 | ||
294 | for (local, arg_kind, ty) in func_params { | |
29967ef6 XL |
295 | // While this is normally an optimization to prevent an unnecessary copy when an argument is |
296 | // not mutated by the current function, this is necessary to support unsized arguments. | |
297 | if let ArgKind::Normal(Some(val)) = arg_kind { | |
298 | if let Some((addr, meta)) = val.try_to_ptr() { | |
94222f64 XL |
299 | // Ownership of the value at the backing storage for an argument is passed to the |
300 | // callee per the ABI, so it is fine to borrow the backing storage of this argument | |
301 | // to prevent a copy. | |
302 | ||
303 | let place = if let Some(meta) = meta { | |
304 | CPlace::for_ptr_with_extra(addr, meta, val.layout()) | |
305 | } else { | |
306 | CPlace::for_ptr(addr, val.layout()) | |
307 | }; | |
308 | ||
309 | self::comments::add_local_place_comments(fx, place, local); | |
310 | ||
311 | assert_eq!(fx.local_map.push(place), local); | |
312 | continue; | |
29967ef6 XL |
313 | } |
314 | } | |
315 | ||
353b0b11 FG |
316 | let layout = fx.layout_of(ty); |
317 | let is_ssa = ssa_analyzed[local].is_ssa(fx, ty); | |
29967ef6 XL |
318 | let place = make_local_place(fx, local, layout, is_ssa); |
319 | assert_eq!(fx.local_map.push(place), local); | |
320 | ||
321 | match arg_kind { | |
322 | ArgKind::Normal(param) => { | |
323 | if let Some(param) = param { | |
324 | place.write_cvalue(fx, param); | |
325 | } | |
326 | } | |
327 | ArgKind::Spread(params) => { | |
328 | for (i, param) in params.into_iter().enumerate() { | |
329 | if let Some(param) = param { | |
353b0b11 | 330 | place.place_field(fx, FieldIdx::new(i)).write_cvalue(fx, param); |
29967ef6 XL |
331 | } |
332 | } | |
333 | } | |
334 | } | |
335 | } | |
336 | ||
337 | for local in fx.mir.vars_and_temps_iter() { | |
fc512014 | 338 | let ty = fx.monomorphize(fx.mir.local_decls[local].ty); |
29967ef6 XL |
339 | let layout = fx.layout_of(ty); |
340 | ||
353b0b11 | 341 | let is_ssa = ssa_analyzed[local].is_ssa(fx, ty); |
29967ef6 XL |
342 | |
343 | let place = make_local_place(fx, local, layout, is_ssa); | |
344 | assert_eq!(fx.local_map.push(place), local); | |
345 | } | |
346 | ||
6a06907d | 347 | fx.bcx.ins().jump(*fx.block_map.get(START_BLOCK).unwrap(), &[]); |
29967ef6 XL |
348 | } |
349 | ||
94222f64 XL |
350 | struct CallArgument<'tcx> { |
351 | value: CValue<'tcx>, | |
352 | is_owned: bool, | |
353 | } | |
354 | ||
355 | // FIXME avoid intermediate `CValue` before calling `adjust_arg_for_abi` | |
356 | fn codegen_call_argument_operand<'tcx>( | |
357 | fx: &mut FunctionCx<'_, '_, 'tcx>, | |
358 | operand: &Operand<'tcx>, | |
359 | ) -> CallArgument<'tcx> { | |
360 | CallArgument { | |
361 | value: codegen_operand(fx, operand), | |
362 | is_owned: matches!(operand, Operand::Move(_)), | |
363 | } | |
364 | } | |
365 | ||
29967ef6 | 366 | pub(crate) fn codegen_terminator_call<'tcx>( |
6a06907d | 367 | fx: &mut FunctionCx<'_, '_, 'tcx>, |
923072b8 | 368 | source_info: mir::SourceInfo, |
29967ef6 XL |
369 | func: &Operand<'tcx>, |
370 | args: &[Operand<'tcx>], | |
923072b8 FG |
371 | destination: Place<'tcx>, |
372 | target: Option<BasicBlock>, | |
29967ef6 | 373 | ) { |
9c376795 FG |
374 | let func = codegen_operand(fx, func); |
375 | let fn_sig = func.layout().ty.fn_sig(fx.tcx); | |
29967ef6 | 376 | |
923072b8 | 377 | let ret_place = codegen_place(fx, destination); |
29967ef6 | 378 | |
f2b60f7d | 379 | // Handle special calls like intrinsics and empty drop glue. |
9c376795 FG |
380 | let instance = if let ty::FnDef(def_id, substs) = *func.layout().ty.kind() { |
381 | let instance = | |
382 | ty::Instance::expect_resolve(fx.tcx, ty::ParamEnv::reveal_all(), def_id, substs) | |
383 | .polymorphize(fx.tcx); | |
29967ef6 XL |
384 | |
385 | if fx.tcx.symbol_name(instance).name.starts_with("llvm.") { | |
386 | crate::intrinsics::codegen_llvm_intrinsic_call( | |
387 | fx, | |
388 | &fx.tcx.symbol_name(instance).name, | |
389 | substs, | |
390 | args, | |
923072b8 FG |
391 | ret_place, |
392 | target, | |
29967ef6 XL |
393 | ); |
394 | return; | |
395 | } | |
396 | ||
397 | match instance.def { | |
398 | InstanceDef::Intrinsic(_) => { | |
923072b8 FG |
399 | crate::intrinsics::codegen_intrinsic_call( |
400 | fx, | |
401 | instance, | |
402 | args, | |
403 | ret_place, | |
404 | target, | |
405 | source_info, | |
406 | ); | |
29967ef6 XL |
407 | return; |
408 | } | |
409 | InstanceDef::DropGlue(_, None) => { | |
410 | // empty drop glue - a nop. | |
923072b8 | 411 | let dest = target.expect("Non terminating drop_in_place_real???"); |
29967ef6 XL |
412 | let ret_block = fx.get_block(dest); |
413 | fx.bcx.ins().jump(ret_block, &[]); | |
414 | return; | |
415 | } | |
416 | _ => Some(instance), | |
417 | } | |
418 | } else { | |
419 | None | |
420 | }; | |
421 | ||
9c376795 | 422 | let extra_args = &args[fn_sig.inputs().skip_binder().len()..]; |
9ffffee4 FG |
423 | let extra_args = fx.tcx.mk_type_list_from_iter( |
424 | extra_args.iter().map(|op_arg| fx.monomorphize(op_arg.ty(fx.mir, fx.tcx))), | |
425 | ); | |
5869c6ff | 426 | let fn_abi = if let Some(instance) = instance { |
c295e0f8 | 427 | RevealAllLayoutCx(fx.tcx).fn_abi_of_instance(instance, extra_args) |
5869c6ff | 428 | } else { |
9c376795 | 429 | RevealAllLayoutCx(fx.tcx).fn_abi_of_fn_ptr(fn_sig, extra_args) |
5869c6ff XL |
430 | }; |
431 | ||
9c376795 | 432 | let is_cold = if fn_sig.abi() == Abi::RustCold { |
064997fb FG |
433 | true |
434 | } else { | |
435 | instance | |
436 | .map(|inst| { | |
437 | fx.tcx.codegen_fn_attrs(inst.def_id()).flags.contains(CodegenFnAttrFlags::COLD) | |
438 | }) | |
439 | .unwrap_or(false) | |
440 | }; | |
29967ef6 | 441 | if is_cold { |
5e7ed085 | 442 | fx.bcx.set_cold_block(fx.bcx.current_block().unwrap()); |
923072b8 | 443 | if let Some(destination_block) = target { |
5e7ed085 FG |
444 | fx.bcx.set_cold_block(fx.get_block(destination_block)); |
445 | } | |
29967ef6 XL |
446 | } |
447 | ||
448 | // Unpack arguments tuple for closures | |
9c376795 | 449 | let mut args = if fn_sig.abi() == Abi::RustCall { |
29967ef6 | 450 | assert_eq!(args.len(), 2, "rust-call abi requires two arguments"); |
94222f64 XL |
451 | let self_arg = codegen_call_argument_operand(fx, &args[0]); |
452 | let pack_arg = codegen_call_argument_operand(fx, &args[1]); | |
29967ef6 | 453 | |
94222f64 | 454 | let tupled_arguments = match pack_arg.value.layout().ty.kind() { |
29967ef6 XL |
455 | ty::Tuple(ref tupled_arguments) => tupled_arguments, |
456 | _ => bug!("argument to function with \"rust-call\" ABI is not a tuple"), | |
457 | }; | |
458 | ||
459 | let mut args = Vec::with_capacity(1 + tupled_arguments.len()); | |
460 | args.push(self_arg); | |
461 | for i in 0..tupled_arguments.len() { | |
94222f64 | 462 | args.push(CallArgument { |
353b0b11 | 463 | value: pack_arg.value.value_field(fx, FieldIdx::new(i)), |
94222f64 XL |
464 | is_owned: pack_arg.is_owned, |
465 | }); | |
29967ef6 XL |
466 | } |
467 | args | |
468 | } else { | |
94222f64 | 469 | args.iter().map(|arg| codegen_call_argument_operand(fx, arg)).collect::<Vec<_>>() |
29967ef6 XL |
470 | }; |
471 | ||
94222f64 XL |
472 | // Pass the caller location for `#[track_caller]`. |
473 | if instance.map(|inst| inst.def.requires_caller_location(fx.tcx)).unwrap_or(false) { | |
923072b8 | 474 | let caller_location = fx.get_caller_location(source_info); |
94222f64 XL |
475 | args.push(CallArgument { value: caller_location, is_owned: false }); |
476 | } | |
477 | ||
478 | let args = args; | |
479 | assert_eq!(fn_abi.args.len(), args.len()); | |
480 | ||
481 | enum CallTarget { | |
482 | Direct(FuncRef), | |
483 | Indirect(SigRef, Value), | |
484 | } | |
485 | ||
486 | let (func_ref, first_arg_override) = match instance { | |
29967ef6 | 487 | // Trait object call |
6a06907d | 488 | Some(Instance { def: InstanceDef::Virtual(_, idx), .. }) => { |
cdc7bbd5 | 489 | if fx.clif_comments.enabled() { |
29967ef6 XL |
490 | let nop_inst = fx.bcx.ins().nop(); |
491 | fx.add_comment( | |
492 | nop_inst, | |
94222f64 | 493 | format!("virtual call; self arg pass mode: {:?}", &fn_abi.args[0]), |
29967ef6 XL |
494 | ); |
495 | } | |
94222f64 XL |
496 | |
497 | let (ptr, method) = crate::vtable::get_ptr_and_method_ref(fx, args[0].value, idx); | |
a2a8927a | 498 | let sig = clif_sig_from_fn_abi(fx.tcx, fx.target_config.default_call_conv, &fn_abi); |
94222f64 XL |
499 | let sig = fx.bcx.import_signature(sig); |
500 | ||
2b03887a | 501 | (CallTarget::Indirect(sig, method), Some(ptr.get_addr(fx))) |
29967ef6 XL |
502 | } |
503 | ||
504 | // Normal call | |
94222f64 XL |
505 | Some(instance) => { |
506 | let func_ref = fx.get_function_ref(instance); | |
507 | (CallTarget::Direct(func_ref), None) | |
508 | } | |
29967ef6 XL |
509 | |
510 | // Indirect call | |
511 | None => { | |
cdc7bbd5 | 512 | if fx.clif_comments.enabled() { |
29967ef6 XL |
513 | let nop_inst = fx.bcx.ins().nop(); |
514 | fx.add_comment(nop_inst, "indirect call"); | |
515 | } | |
94222f64 | 516 | |
9c376795 | 517 | let func = func.load_scalar(fx); |
a2a8927a | 518 | let sig = clif_sig_from_fn_abi(fx.tcx, fx.target_config.default_call_conv, &fn_abi); |
94222f64 XL |
519 | let sig = fx.bcx.import_signature(sig); |
520 | ||
521 | (CallTarget::Indirect(sig, func), None) | |
29967ef6 XL |
522 | } |
523 | }; | |
524 | ||
94222f64 XL |
525 | self::returning::codegen_with_call_return_arg(fx, &fn_abi.ret, ret_place, |fx, return_ptr| { |
526 | let call_args = return_ptr | |
527 | .into_iter() | |
528 | .chain(first_arg_override.into_iter()) | |
529 | .chain( | |
530 | args.into_iter() | |
531 | .enumerate() | |
532 | .skip(if first_arg_override.is_some() { 1 } else { 0 }) | |
353b0b11 | 533 | .flat_map(|(i, arg)| { |
94222f64 | 534 | adjust_arg_for_abi(fx, arg.value, &fn_abi.args[i], arg.is_owned).into_iter() |
353b0b11 | 535 | }), |
94222f64 XL |
536 | ) |
537 | .collect::<Vec<Value>>(); | |
538 | ||
539 | let call_inst = match func_ref { | |
540 | CallTarget::Direct(func_ref) => fx.bcx.ins().call(func_ref, &call_args), | |
541 | CallTarget::Indirect(sig, func_ptr) => { | |
542 | fx.bcx.ins().call_indirect(sig, func_ptr, &call_args) | |
29967ef6 | 543 | } |
94222f64 | 544 | }; |
29967ef6 | 545 | |
94222f64 | 546 | // FIXME find a cleaner way to support varargs |
9c376795 FG |
547 | if fn_sig.c_variadic() { |
548 | if !matches!(fn_sig.abi(), Abi::C { .. }) { | |
923072b8 FG |
549 | fx.tcx.sess.span_fatal( |
550 | source_info.span, | |
9c376795 | 551 | &format!("Variadic call for non-C abi {:?}", fn_sig.abi()), |
923072b8 | 552 | ); |
94222f64 XL |
553 | } |
554 | let sig_ref = fx.bcx.func.dfg.call_signature(call_inst).unwrap(); | |
555 | let abi_params = call_args | |
556 | .into_iter() | |
557 | .map(|arg| { | |
558 | let ty = fx.bcx.func.dfg.value_type(arg); | |
559 | if !ty.is_int() { | |
560 | // FIXME set %al to upperbound on float args once floats are supported | |
923072b8 FG |
561 | fx.tcx.sess.span_fatal( |
562 | source_info.span, | |
563 | &format!("Non int ty {:?} for variadic call", ty), | |
564 | ); | |
94222f64 XL |
565 | } |
566 | AbiParam::new(ty) | |
567 | }) | |
568 | .collect::<Vec<AbiParam>>(); | |
569 | fx.bcx.func.dfg.signatures[sig_ref].params = abi_params; | |
29967ef6 | 570 | } |
94222f64 XL |
571 | |
572 | call_inst | |
573 | }); | |
29967ef6 | 574 | |
923072b8 | 575 | if let Some(dest) = target { |
29967ef6 XL |
576 | let ret_block = fx.get_block(dest); |
577 | fx.bcx.ins().jump(ret_block, &[]); | |
578 | } else { | |
5e7ed085 | 579 | fx.bcx.ins().trap(TrapCode::UnreachableCodeReached); |
29967ef6 XL |
580 | } |
581 | } | |
582 | ||
583 | pub(crate) fn codegen_drop<'tcx>( | |
6a06907d | 584 | fx: &mut FunctionCx<'_, '_, 'tcx>, |
923072b8 | 585 | source_info: mir::SourceInfo, |
29967ef6 XL |
586 | drop_place: CPlace<'tcx>, |
587 | ) { | |
588 | let ty = drop_place.layout().ty; | |
5869c6ff | 589 | let drop_instance = Instance::resolve_drop_in_place(fx.tcx, ty).polymorphize(fx.tcx); |
29967ef6 | 590 | |
5869c6ff | 591 | if let ty::InstanceDef::DropGlue(_, None) = drop_instance.def { |
29967ef6 XL |
592 | // we don't actually need to drop anything |
593 | } else { | |
29967ef6 | 594 | match ty.kind() { |
2b03887a FG |
595 | ty::Dynamic(_, _, ty::Dyn) => { |
596 | // IN THIS ARM, WE HAVE: | |
597 | // ty = *mut (dyn Trait) | |
598 | // which is: exists<T> ( *mut T, Vtable<T: Trait> ) | |
599 | // args[0] args[1] | |
600 | // | |
601 | // args = ( Data, Vtable ) | |
602 | // | | |
603 | // v | |
604 | // /-------\ | |
605 | // | ... | | |
606 | // \-------/ | |
607 | // | |
29967ef6 XL |
608 | let (ptr, vtable) = drop_place.to_ptr_maybe_unsized(); |
609 | let ptr = ptr.get_addr(fx); | |
610 | let drop_fn = crate::vtable::drop_fn_of_obj(fx, vtable.unwrap()); | |
611 | ||
5869c6ff XL |
612 | // FIXME(eddyb) perhaps move some of this logic into |
613 | // `Instance::resolve_drop_in_place`? | |
614 | let virtual_drop = Instance { | |
615 | def: ty::InstanceDef::Virtual(drop_instance.def_id(), 0), | |
616 | substs: drop_instance.substs, | |
617 | }; | |
c295e0f8 XL |
618 | let fn_abi = |
619 | RevealAllLayoutCx(fx.tcx).fn_abi_of_instance(virtual_drop, ty::List::empty()); | |
5869c6ff | 620 | |
a2a8927a | 621 | let sig = clif_sig_from_fn_abi(fx.tcx, fx.target_config.default_call_conv, &fn_abi); |
29967ef6 XL |
622 | let sig = fx.bcx.import_signature(sig); |
623 | fx.bcx.ins().call_indirect(sig, drop_fn, &[ptr]); | |
624 | } | |
2b03887a FG |
625 | ty::Dynamic(_, _, ty::DynStar) => { |
626 | // IN THIS ARM, WE HAVE: | |
627 | // ty = *mut (dyn* Trait) | |
628 | // which is: *mut exists<T: sizeof(T) == sizeof(usize)> (T, Vtable<T: Trait>) | |
629 | // | |
630 | // args = [ * ] | |
631 | // | | |
632 | // v | |
633 | // ( Data, Vtable ) | |
634 | // | | |
635 | // v | |
636 | // /-------\ | |
637 | // | ... | | |
638 | // \-------/ | |
639 | // | |
640 | // | |
641 | // WE CAN CONVERT THIS INTO THE ABOVE LOGIC BY DOING | |
642 | // | |
643 | // data = &(*args[0]).0 // gives a pointer to Data above (really the same pointer) | |
644 | // vtable = (*args[0]).1 // loads the vtable out | |
645 | // (data, vtable) // an equivalent Rust `*mut dyn Trait` | |
646 | // | |
647 | // SO THEN WE CAN USE THE ABOVE CODE. | |
648 | let (data, vtable) = drop_place.to_cvalue(fx).dyn_star_force_data_on_stack(fx); | |
649 | let drop_fn = crate::vtable::drop_fn_of_obj(fx, vtable); | |
650 | ||
651 | let virtual_drop = Instance { | |
652 | def: ty::InstanceDef::Virtual(drop_instance.def_id(), 0), | |
653 | substs: drop_instance.substs, | |
654 | }; | |
655 | let fn_abi = | |
656 | RevealAllLayoutCx(fx.tcx).fn_abi_of_instance(virtual_drop, ty::List::empty()); | |
657 | ||
658 | let sig = clif_sig_from_fn_abi(fx.tcx, fx.target_config.default_call_conv, &fn_abi); | |
659 | let sig = fx.bcx.import_signature(sig); | |
660 | fx.bcx.ins().call_indirect(sig, drop_fn, &[data]); | |
661 | } | |
29967ef6 | 662 | _ => { |
5869c6ff XL |
663 | assert!(!matches!(drop_instance.def, InstanceDef::Virtual(_, _))); |
664 | ||
c295e0f8 XL |
665 | let fn_abi = |
666 | RevealAllLayoutCx(fx.tcx).fn_abi_of_instance(drop_instance, ty::List::empty()); | |
29967ef6 XL |
667 | |
668 | let arg_value = drop_place.place_ref( | |
669 | fx, | |
670 | fx.layout_of(fx.tcx.mk_ref( | |
5099ac24 | 671 | fx.tcx.lifetimes.re_erased, |
6a06907d | 672 | TypeAndMut { ty, mutbl: crate::rustc_hir::Mutability::Mut }, |
29967ef6 XL |
673 | )), |
674 | ); | |
94222f64 | 675 | let arg_value = adjust_arg_for_abi(fx, arg_value, &fn_abi.args[0], true); |
29967ef6 XL |
676 | |
677 | let mut call_args: Vec<Value> = arg_value.into_iter().collect::<Vec<_>>(); | |
678 | ||
5869c6ff | 679 | if drop_instance.def.requires_caller_location(fx.tcx) { |
29967ef6 | 680 | // Pass the caller location for `#[track_caller]`. |
923072b8 | 681 | let caller_location = fx.get_caller_location(source_info); |
5869c6ff | 682 | call_args.extend( |
94222f64 | 683 | adjust_arg_for_abi(fx, caller_location, &fn_abi.args[1], false).into_iter(), |
5869c6ff | 684 | ); |
29967ef6 XL |
685 | } |
686 | ||
5869c6ff | 687 | let func_ref = fx.get_function_ref(drop_instance); |
29967ef6 XL |
688 | fx.bcx.ins().call(func_ref, &call_args); |
689 | } | |
690 | } | |
691 | } | |
692 | } |