]>
Commit | Line | Data |
---|---|---|
29967ef6 XL |
1 | //! Argument passing |
2 | ||
3 | use crate::prelude::*; | |
5869c6ff | 4 | use crate::value_and_place::assert_assignable; |
29967ef6 | 5 | |
5869c6ff XL |
6 | use cranelift_codegen::ir::{ArgumentExtension, ArgumentPurpose}; |
7 | use rustc_target::abi::call::{ | |
8 | ArgAbi, ArgAttributes, ArgExtension as RustcArgExtension, CastTarget, PassMode, Reg, RegKind, | |
9 | }; | |
10 | use smallvec::{smallvec, SmallVec}; | |
29967ef6 | 11 | |
5869c6ff XL |
12 | pub(super) trait ArgAbiExt<'tcx> { |
13 | fn get_abi_param(&self, tcx: TyCtxt<'tcx>) -> SmallVec<[AbiParam; 2]>; | |
14 | fn get_abi_return(&self, tcx: TyCtxt<'tcx>) -> (Option<AbiParam>, Vec<AbiParam>); | |
29967ef6 XL |
15 | } |
16 | ||
5869c6ff XL |
17 | fn reg_to_abi_param(reg: Reg) -> AbiParam { |
18 | let clif_ty = match (reg.kind, reg.size.bytes()) { | |
19 | (RegKind::Integer, 1) => types::I8, | |
20 | (RegKind::Integer, 2) => types::I16, | |
21 | (RegKind::Integer, 4) => types::I32, | |
22 | (RegKind::Integer, 8) => types::I64, | |
23 | (RegKind::Integer, 16) => types::I128, | |
24 | (RegKind::Float, 4) => types::F32, | |
25 | (RegKind::Float, 8) => types::F64, | |
26 | (RegKind::Vector, size) => types::I8.by(u16::try_from(size).unwrap()).unwrap(), | |
27 | _ => unreachable!("{:?}", reg), | |
28 | }; | |
29 | AbiParam::new(clif_ty) | |
29967ef6 XL |
30 | } |
31 | ||
5869c6ff XL |
32 | fn apply_arg_attrs_to_abi_param(mut param: AbiParam, arg_attrs: ArgAttributes) -> AbiParam { |
33 | match arg_attrs.arg_ext { | |
34 | RustcArgExtension::None => {} | |
35 | RustcArgExtension::Zext => param.extension = ArgumentExtension::Uext, | |
36 | RustcArgExtension::Sext => param.extension = ArgumentExtension::Sext, | |
29967ef6 | 37 | } |
5869c6ff | 38 | param |
29967ef6 XL |
39 | } |
40 | ||
5869c6ff XL |
41 | fn cast_target_to_abi_params(cast: CastTarget) -> SmallVec<[AbiParam; 2]> { |
42 | let (rest_count, rem_bytes) = if cast.rest.unit.size.bytes() == 0 { | |
43 | (0, 0) | |
44 | } else { | |
45 | ( | |
46 | cast.rest.total.bytes() / cast.rest.unit.size.bytes(), | |
47 | cast.rest.total.bytes() % cast.rest.unit.size.bytes(), | |
48 | ) | |
49 | }; | |
29967ef6 | 50 | |
5869c6ff XL |
51 | if cast.prefix.iter().all(|x| x.is_none()) { |
52 | // Simplify to a single unit when there is no prefix and size <= unit size | |
53 | if cast.rest.total <= cast.rest.unit.size { | |
54 | let clif_ty = match (cast.rest.unit.kind, cast.rest.unit.size.bytes()) { | |
55 | (RegKind::Integer, 1) => types::I8, | |
56 | (RegKind::Integer, 2) => types::I16, | |
57 | (RegKind::Integer, 3..=4) => types::I32, | |
58 | (RegKind::Integer, 5..=8) => types::I64, | |
59 | (RegKind::Integer, 9..=16) => types::I128, | |
60 | (RegKind::Float, 4) => types::F32, | |
61 | (RegKind::Float, 8) => types::F64, | |
62 | (RegKind::Vector, size) => types::I8.by(u16::try_from(size).unwrap()).unwrap(), | |
63 | _ => unreachable!("{:?}", cast.rest.unit), | |
64 | }; | |
65 | return smallvec![AbiParam::new(clif_ty)]; | |
29967ef6 XL |
66 | } |
67 | } | |
29967ef6 | 68 | |
5869c6ff XL |
69 | // Create list of fields in the main structure |
70 | let mut args = cast | |
71 | .prefix | |
72 | .iter() | |
73 | .flatten() | |
a2a8927a | 74 | .map(|®| reg_to_abi_param(reg)) |
5869c6ff XL |
75 | .chain((0..rest_count).map(|_| reg_to_abi_param(cast.rest.unit))) |
76 | .collect::<SmallVec<_>>(); | |
29967ef6 | 77 | |
5869c6ff XL |
78 | // Append final integer |
79 | if rem_bytes != 0 { | |
80 | // Only integers can be really split further. | |
81 | assert_eq!(cast.rest.unit.kind, RegKind::Integer); | |
82 | args.push(reg_to_abi_param(Reg { | |
83 | kind: RegKind::Integer, | |
84 | size: Size::from_bytes(rem_bytes), | |
85 | })); | |
29967ef6 | 86 | } |
29967ef6 | 87 | |
5869c6ff | 88 | args |
29967ef6 XL |
89 | } |
90 | ||
5869c6ff XL |
91 | impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { |
92 | fn get_abi_param(&self, tcx: TyCtxt<'tcx>) -> SmallVec<[AbiParam; 2]> { | |
93 | match self.mode { | |
94 | PassMode::Ignore => smallvec![], | |
c295e0f8 | 95 | PassMode::Direct(attrs) => match self.layout.abi { |
6a06907d | 96 | Abi::Scalar(scalar) => smallvec![apply_arg_attrs_to_abi_param( |
c295e0f8 | 97 | AbiParam::new(scalar_to_clif_type(tcx, scalar)), |
6a06907d XL |
98 | attrs |
99 | )], | |
5869c6ff XL |
100 | Abi::Vector { .. } => { |
101 | let vector_ty = crate::intrinsics::clif_vector_type(tcx, self.layout).unwrap(); | |
102 | smallvec![AbiParam::new(vector_ty)] | |
103 | } | |
104 | _ => unreachable!("{:?}", self.layout.abi), | |
105 | }, | |
c295e0f8 | 106 | PassMode::Pair(attrs_a, attrs_b) => match self.layout.abi { |
5869c6ff | 107 | Abi::ScalarPair(a, b) => { |
c295e0f8 XL |
108 | let a = scalar_to_clif_type(tcx, a); |
109 | let b = scalar_to_clif_type(tcx, b); | |
5869c6ff XL |
110 | smallvec![ |
111 | apply_arg_attrs_to_abi_param(AbiParam::new(a), attrs_a), | |
112 | apply_arg_attrs_to_abi_param(AbiParam::new(b), attrs_b), | |
113 | ] | |
114 | } | |
115 | _ => unreachable!("{:?}", self.layout.abi), | |
116 | }, | |
117 | PassMode::Cast(cast) => cast_target_to_abi_params(cast), | |
6a06907d | 118 | PassMode::Indirect { attrs, extra_attrs: None, on_stack } => { |
5869c6ff | 119 | if on_stack { |
a2a8927a XL |
120 | // Abi requires aligning struct size to pointer size |
121 | let size = self.layout.size.align_to(tcx.data_layout.pointer_align.abi); | |
122 | let size = u32::try_from(size.bytes()).unwrap(); | |
5869c6ff XL |
123 | smallvec![apply_arg_attrs_to_abi_param( |
124 | AbiParam::special(pointer_ty(tcx), ArgumentPurpose::StructArgument(size),), | |
125 | attrs | |
126 | )] | |
29967ef6 | 127 | } else { |
6a06907d | 128 | smallvec![apply_arg_attrs_to_abi_param(AbiParam::new(pointer_ty(tcx)), attrs)] |
29967ef6 XL |
129 | } |
130 | } | |
6a06907d | 131 | PassMode::Indirect { attrs, extra_attrs: Some(extra_attrs), on_stack } => { |
5869c6ff XL |
132 | assert!(!on_stack); |
133 | smallvec![ | |
134 | apply_arg_attrs_to_abi_param(AbiParam::new(pointer_ty(tcx)), attrs), | |
135 | apply_arg_attrs_to_abi_param(AbiParam::new(pointer_ty(tcx)), extra_attrs), | |
136 | ] | |
137 | } | |
138 | } | |
139 | } | |
29967ef6 | 140 | |
5869c6ff XL |
141 | fn get_abi_return(&self, tcx: TyCtxt<'tcx>) -> (Option<AbiParam>, Vec<AbiParam>) { |
142 | match self.mode { | |
143 | PassMode::Ignore => (None, vec![]), | |
c295e0f8 | 144 | PassMode::Direct(_) => match self.layout.abi { |
6a06907d | 145 | Abi::Scalar(scalar) => { |
c295e0f8 | 146 | (None, vec![AbiParam::new(scalar_to_clif_type(tcx, scalar))]) |
6a06907d | 147 | } |
5869c6ff XL |
148 | Abi::Vector { .. } => { |
149 | let vector_ty = crate::intrinsics::clif_vector_type(tcx, self.layout).unwrap(); | |
150 | (None, vec![AbiParam::new(vector_ty)]) | |
151 | } | |
152 | _ => unreachable!("{:?}", self.layout.abi), | |
153 | }, | |
c295e0f8 | 154 | PassMode::Pair(_, _) => match self.layout.abi { |
5869c6ff | 155 | Abi::ScalarPair(a, b) => { |
c295e0f8 XL |
156 | let a = scalar_to_clif_type(tcx, a); |
157 | let b = scalar_to_clif_type(tcx, b); | |
5869c6ff XL |
158 | (None, vec![AbiParam::new(a), AbiParam::new(b)]) |
159 | } | |
160 | _ => unreachable!("{:?}", self.layout.abi), | |
29967ef6 | 161 | }, |
5869c6ff | 162 | PassMode::Cast(cast) => (None, cast_target_to_abi_params(cast).into_iter().collect()), |
6a06907d | 163 | PassMode::Indirect { attrs: _, extra_attrs: None, on_stack } => { |
5869c6ff | 164 | assert!(!on_stack); |
6a06907d XL |
165 | (Some(AbiParam::special(pointer_ty(tcx), ArgumentPurpose::StructReturn)), vec![]) |
166 | } | |
167 | PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => { | |
168 | unreachable!("unsized return value") | |
5869c6ff | 169 | } |
29967ef6 XL |
170 | } |
171 | } | |
172 | } | |
173 | ||
5869c6ff | 174 | pub(super) fn to_casted_value<'tcx>( |
6a06907d | 175 | fx: &mut FunctionCx<'_, '_, 'tcx>, |
5869c6ff XL |
176 | arg: CValue<'tcx>, |
177 | cast: CastTarget, | |
178 | ) -> SmallVec<[Value; 2]> { | |
179 | let (ptr, meta) = arg.force_stack(fx); | |
180 | assert!(meta.is_none()); | |
181 | let mut offset = 0; | |
182 | cast_target_to_abi_params(cast) | |
183 | .into_iter() | |
184 | .map(|param| { | |
6a06907d | 185 | let val = ptr.offset_i64(fx, offset).load(fx, param.value_type, MemFlags::new()); |
5869c6ff XL |
186 | offset += i64::from(param.value_type.bytes()); |
187 | val | |
188 | }) | |
189 | .collect() | |
190 | } | |
191 | ||
192 | pub(super) fn from_casted_value<'tcx>( | |
6a06907d | 193 | fx: &mut FunctionCx<'_, '_, 'tcx>, |
5869c6ff XL |
194 | block_params: &[Value], |
195 | layout: TyAndLayout<'tcx>, | |
196 | cast: CastTarget, | |
197 | ) -> CValue<'tcx> { | |
198 | let abi_params = cast_target_to_abi_params(cast); | |
6a06907d | 199 | let abi_param_size: u32 = abi_params.iter().map(|param| param.value_type.bytes()).sum(); |
5869c6ff XL |
200 | let layout_size = u32::try_from(layout.size.bytes()).unwrap(); |
201 | let stack_slot = fx.bcx.create_stack_slot(StackSlotData { | |
202 | kind: StackSlotKind::ExplicitSlot, | |
203 | // FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to | |
204 | // specify stack slot alignment. | |
205 | // Stack slot size may be bigger for for example `[u8; 3]` which is packed into an `i32`. | |
206 | // It may also be smaller for example when the type is a wrapper around an integer with a | |
207 | // larger alignment than the integer. | |
208 | size: (std::cmp::max(abi_param_size, layout_size) + 15) / 16 * 16, | |
5869c6ff XL |
209 | }); |
210 | let ptr = Pointer::new(fx.bcx.ins().stack_addr(pointer_ty(fx.tcx), stack_slot, 0)); | |
211 | let mut offset = 0; | |
cdc7bbd5 | 212 | let mut block_params_iter = block_params.iter().copied(); |
5869c6ff XL |
213 | for param in abi_params { |
214 | let val = ptr.offset_i64(fx, offset).store( | |
215 | fx, | |
216 | block_params_iter.next().unwrap(), | |
217 | MemFlags::new(), | |
218 | ); | |
219 | offset += i64::from(param.value_type.bytes()); | |
220 | val | |
221 | } | |
222 | assert_eq!(block_params_iter.next(), None, "Leftover block param"); | |
223 | CValue::by_ref(ptr, layout) | |
224 | } | |
225 | ||
29967ef6 XL |
226 | /// Get a set of values to be passed as function arguments. |
227 | pub(super) fn adjust_arg_for_abi<'tcx>( | |
6a06907d | 228 | fx: &mut FunctionCx<'_, '_, 'tcx>, |
29967ef6 | 229 | arg: CValue<'tcx>, |
5869c6ff | 230 | arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, |
94222f64 | 231 | is_owned: bool, |
5869c6ff XL |
232 | ) -> SmallVec<[Value; 2]> { |
233 | assert_assignable(fx, arg.layout().ty, arg_abi.layout.ty); | |
234 | match arg_abi.mode { | |
235 | PassMode::Ignore => smallvec![], | |
236 | PassMode::Direct(_) => smallvec![arg.load_scalar(fx)], | |
237 | PassMode::Pair(_, _) => { | |
29967ef6 | 238 | let (a, b) = arg.load_scalar_pair(fx); |
5869c6ff | 239 | smallvec![a, b] |
29967ef6 | 240 | } |
5869c6ff | 241 | PassMode::Cast(cast) => to_casted_value(fx, arg, cast), |
94222f64 XL |
242 | PassMode::Indirect { .. } => { |
243 | if is_owned { | |
244 | match arg.force_stack(fx) { | |
245 | (ptr, None) => smallvec![ptr.get_addr(fx)], | |
246 | (ptr, Some(meta)) => smallvec![ptr.get_addr(fx), meta], | |
247 | } | |
248 | } else { | |
249 | // Ownership of the value at the backing storage for an argument is passed to the | |
250 | // callee per the ABI, so we must make a copy of the argument unless the argument | |
251 | // local is moved. | |
252 | let place = CPlace::new_stack_slot(fx, arg.layout()); | |
253 | place.write_cvalue(fx, arg); | |
254 | smallvec![place.to_ptr().get_addr(fx)] | |
255 | } | |
256 | } | |
29967ef6 XL |
257 | } |
258 | } | |
259 | ||
260 | /// Create a [`CValue`] containing the value of a function parameter adding clif function parameters | |
261 | /// as necessary. | |
262 | pub(super) fn cvalue_for_param<'tcx>( | |
6a06907d | 263 | fx: &mut FunctionCx<'_, '_, 'tcx>, |
cdc7bbd5 XL |
264 | local: Option<mir::Local>, |
265 | local_field: Option<usize>, | |
5869c6ff XL |
266 | arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, |
267 | block_params_iter: &mut impl Iterator<Item = Value>, | |
29967ef6 | 268 | ) -> Option<CValue<'tcx>> { |
5869c6ff XL |
269 | let block_params = arg_abi |
270 | .get_abi_param(fx.tcx) | |
271 | .into_iter() | |
272 | .map(|abi_param| { | |
273 | let block_param = block_params_iter.next().unwrap(); | |
6a06907d | 274 | assert_eq!(fx.bcx.func.dfg.value_type(block_param), abi_param.value_type); |
5869c6ff XL |
275 | block_param |
276 | }) | |
277 | .collect::<SmallVec<[_; 2]>>(); | |
29967ef6 | 278 | |
29967ef6 XL |
279 | crate::abi::comments::add_arg_comment( |
280 | fx, | |
281 | "arg", | |
282 | local, | |
283 | local_field, | |
5869c6ff XL |
284 | &block_params, |
285 | arg_abi.mode, | |
286 | arg_abi.layout, | |
29967ef6 XL |
287 | ); |
288 | ||
5869c6ff XL |
289 | match arg_abi.mode { |
290 | PassMode::Ignore => None, | |
291 | PassMode::Direct(_) => { | |
292 | assert_eq!(block_params.len(), 1, "{:?}", block_params); | |
293 | Some(CValue::by_val(block_params[0], arg_abi.layout)) | |
294 | } | |
295 | PassMode::Pair(_, _) => { | |
296 | assert_eq!(block_params.len(), 2, "{:?}", block_params); | |
6a06907d | 297 | Some(CValue::by_val_pair(block_params[0], block_params[1], arg_abi.layout)) |
5869c6ff XL |
298 | } |
299 | PassMode::Cast(cast) => Some(from_casted_value(fx, &block_params, arg_abi.layout, cast)), | |
6a06907d | 300 | PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } => { |
5869c6ff | 301 | assert_eq!(block_params.len(), 1, "{:?}", block_params); |
6a06907d | 302 | Some(CValue::by_ref(Pointer::new(block_params[0]), arg_abi.layout)) |
29967ef6 | 303 | } |
6a06907d | 304 | PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => { |
5869c6ff XL |
305 | assert_eq!(block_params.len(), 2, "{:?}", block_params); |
306 | Some(CValue::by_ref_unsized( | |
307 | Pointer::new(block_params[0]), | |
308 | block_params[1], | |
309 | arg_abi.layout, | |
310 | )) | |
29967ef6 XL |
311 | } |
312 | } | |
313 | } |