]>
Commit | Line | Data |
---|---|---|
29967ef6 XL |
1 | //! Argument passing |
2 | ||
3 | use crate::prelude::*; | |
5869c6ff | 4 | use crate::value_and_place::assert_assignable; |
29967ef6 | 5 | |
5869c6ff XL |
6 | use cranelift_codegen::ir::{ArgumentExtension, ArgumentPurpose}; |
7 | use rustc_target::abi::call::{ | |
8 | ArgAbi, ArgAttributes, ArgExtension as RustcArgExtension, CastTarget, PassMode, Reg, RegKind, | |
9 | }; | |
10 | use smallvec::{smallvec, SmallVec}; | |
29967ef6 | 11 | |
5869c6ff XL |
12 | pub(super) trait ArgAbiExt<'tcx> { |
13 | fn get_abi_param(&self, tcx: TyCtxt<'tcx>) -> SmallVec<[AbiParam; 2]>; | |
14 | fn get_abi_return(&self, tcx: TyCtxt<'tcx>) -> (Option<AbiParam>, Vec<AbiParam>); | |
29967ef6 XL |
15 | } |
16 | ||
5869c6ff XL |
17 | fn reg_to_abi_param(reg: Reg) -> AbiParam { |
18 | let clif_ty = match (reg.kind, reg.size.bytes()) { | |
19 | (RegKind::Integer, 1) => types::I8, | |
20 | (RegKind::Integer, 2) => types::I16, | |
064997fb FG |
21 | (RegKind::Integer, 3..=4) => types::I32, |
22 | (RegKind::Integer, 5..=8) => types::I64, | |
23 | (RegKind::Integer, 9..=16) => types::I128, | |
5869c6ff XL |
24 | (RegKind::Float, 4) => types::F32, |
25 | (RegKind::Float, 8) => types::F64, | |
f2b60f7d | 26 | (RegKind::Vector, size) => types::I8.by(u32::try_from(size).unwrap()).unwrap(), |
5869c6ff XL |
27 | _ => unreachable!("{:?}", reg), |
28 | }; | |
29 | AbiParam::new(clif_ty) | |
29967ef6 XL |
30 | } |
31 | ||
5869c6ff XL |
32 | fn apply_arg_attrs_to_abi_param(mut param: AbiParam, arg_attrs: ArgAttributes) -> AbiParam { |
33 | match arg_attrs.arg_ext { | |
34 | RustcArgExtension::None => {} | |
35 | RustcArgExtension::Zext => param.extension = ArgumentExtension::Uext, | |
36 | RustcArgExtension::Sext => param.extension = ArgumentExtension::Sext, | |
29967ef6 | 37 | } |
5869c6ff | 38 | param |
29967ef6 XL |
39 | } |
40 | ||
f2b60f7d | 41 | fn cast_target_to_abi_params(cast: &CastTarget) -> SmallVec<[AbiParam; 2]> { |
5869c6ff XL |
42 | let (rest_count, rem_bytes) = if cast.rest.unit.size.bytes() == 0 { |
43 | (0, 0) | |
44 | } else { | |
45 | ( | |
46 | cast.rest.total.bytes() / cast.rest.unit.size.bytes(), | |
47 | cast.rest.total.bytes() % cast.rest.unit.size.bytes(), | |
48 | ) | |
49 | }; | |
29967ef6 | 50 | |
064997fb FG |
51 | // Note: Unlike the LLVM equivalent of this code we don't have separate branches for when there |
52 | // is no prefix as a single unit, an array and a heterogeneous struct are not represented using | |
53 | // different types in Cranelift IR. Instead a single array of primitive types is used. | |
29967ef6 | 54 | |
5869c6ff XL |
55 | // Create list of fields in the main structure |
56 | let mut args = cast | |
57 | .prefix | |
58 | .iter() | |
59 | .flatten() | |
a2a8927a | 60 | .map(|®| reg_to_abi_param(reg)) |
5869c6ff XL |
61 | .chain((0..rest_count).map(|_| reg_to_abi_param(cast.rest.unit))) |
62 | .collect::<SmallVec<_>>(); | |
29967ef6 | 63 | |
5869c6ff XL |
64 | // Append final integer |
65 | if rem_bytes != 0 { | |
66 | // Only integers can be really split further. | |
67 | assert_eq!(cast.rest.unit.kind, RegKind::Integer); | |
68 | args.push(reg_to_abi_param(Reg { | |
69 | kind: RegKind::Integer, | |
70 | size: Size::from_bytes(rem_bytes), | |
71 | })); | |
29967ef6 | 72 | } |
29967ef6 | 73 | |
5869c6ff | 74 | args |
29967ef6 XL |
75 | } |
76 | ||
5869c6ff XL |
77 | impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { |
78 | fn get_abi_param(&self, tcx: TyCtxt<'tcx>) -> SmallVec<[AbiParam; 2]> { | |
79 | match self.mode { | |
80 | PassMode::Ignore => smallvec![], | |
c295e0f8 | 81 | PassMode::Direct(attrs) => match self.layout.abi { |
6a06907d | 82 | Abi::Scalar(scalar) => smallvec![apply_arg_attrs_to_abi_param( |
c295e0f8 | 83 | AbiParam::new(scalar_to_clif_type(tcx, scalar)), |
6a06907d XL |
84 | attrs |
85 | )], | |
5869c6ff | 86 | Abi::Vector { .. } => { |
49aad941 | 87 | let vector_ty = crate::intrinsics::clif_vector_type(tcx, self.layout); |
5869c6ff XL |
88 | smallvec![AbiParam::new(vector_ty)] |
89 | } | |
90 | _ => unreachable!("{:?}", self.layout.abi), | |
91 | }, | |
c295e0f8 | 92 | PassMode::Pair(attrs_a, attrs_b) => match self.layout.abi { |
5869c6ff | 93 | Abi::ScalarPair(a, b) => { |
c295e0f8 XL |
94 | let a = scalar_to_clif_type(tcx, a); |
95 | let b = scalar_to_clif_type(tcx, b); | |
5869c6ff XL |
96 | smallvec![ |
97 | apply_arg_attrs_to_abi_param(AbiParam::new(a), attrs_a), | |
98 | apply_arg_attrs_to_abi_param(AbiParam::new(b), attrs_b), | |
99 | ] | |
100 | } | |
101 | _ => unreachable!("{:?}", self.layout.abi), | |
102 | }, | |
f2b60f7d FG |
103 | PassMode::Cast(ref cast, pad_i32) => { |
104 | assert!(!pad_i32, "padding support not yet implemented"); | |
105 | cast_target_to_abi_params(cast) | |
106 | } | |
6a06907d | 107 | PassMode::Indirect { attrs, extra_attrs: None, on_stack } => { |
5869c6ff | 108 | if on_stack { |
a2a8927a XL |
109 | // Abi requires aligning struct size to pointer size |
110 | let size = self.layout.size.align_to(tcx.data_layout.pointer_align.abi); | |
111 | let size = u32::try_from(size.bytes()).unwrap(); | |
5869c6ff XL |
112 | smallvec![apply_arg_attrs_to_abi_param( |
113 | AbiParam::special(pointer_ty(tcx), ArgumentPurpose::StructArgument(size),), | |
114 | attrs | |
115 | )] | |
29967ef6 | 116 | } else { |
6a06907d | 117 | smallvec![apply_arg_attrs_to_abi_param(AbiParam::new(pointer_ty(tcx)), attrs)] |
29967ef6 XL |
118 | } |
119 | } | |
6a06907d | 120 | PassMode::Indirect { attrs, extra_attrs: Some(extra_attrs), on_stack } => { |
5869c6ff XL |
121 | assert!(!on_stack); |
122 | smallvec![ | |
123 | apply_arg_attrs_to_abi_param(AbiParam::new(pointer_ty(tcx)), attrs), | |
124 | apply_arg_attrs_to_abi_param(AbiParam::new(pointer_ty(tcx)), extra_attrs), | |
125 | ] | |
126 | } | |
127 | } | |
128 | } | |
29967ef6 | 129 | |
5869c6ff XL |
130 | fn get_abi_return(&self, tcx: TyCtxt<'tcx>) -> (Option<AbiParam>, Vec<AbiParam>) { |
131 | match self.mode { | |
132 | PassMode::Ignore => (None, vec![]), | |
c295e0f8 | 133 | PassMode::Direct(_) => match self.layout.abi { |
6a06907d | 134 | Abi::Scalar(scalar) => { |
c295e0f8 | 135 | (None, vec![AbiParam::new(scalar_to_clif_type(tcx, scalar))]) |
6a06907d | 136 | } |
5869c6ff | 137 | Abi::Vector { .. } => { |
49aad941 | 138 | let vector_ty = crate::intrinsics::clif_vector_type(tcx, self.layout); |
5869c6ff XL |
139 | (None, vec![AbiParam::new(vector_ty)]) |
140 | } | |
141 | _ => unreachable!("{:?}", self.layout.abi), | |
142 | }, | |
c295e0f8 | 143 | PassMode::Pair(_, _) => match self.layout.abi { |
5869c6ff | 144 | Abi::ScalarPair(a, b) => { |
c295e0f8 XL |
145 | let a = scalar_to_clif_type(tcx, a); |
146 | let b = scalar_to_clif_type(tcx, b); | |
5869c6ff XL |
147 | (None, vec![AbiParam::new(a), AbiParam::new(b)]) |
148 | } | |
149 | _ => unreachable!("{:?}", self.layout.abi), | |
29967ef6 | 150 | }, |
f2b60f7d FG |
151 | PassMode::Cast(ref cast, _) => { |
152 | (None, cast_target_to_abi_params(cast).into_iter().collect()) | |
153 | } | |
6a06907d | 154 | PassMode::Indirect { attrs: _, extra_attrs: None, on_stack } => { |
5869c6ff | 155 | assert!(!on_stack); |
6a06907d XL |
156 | (Some(AbiParam::special(pointer_ty(tcx), ArgumentPurpose::StructReturn)), vec![]) |
157 | } | |
158 | PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => { | |
159 | unreachable!("unsized return value") | |
5869c6ff | 160 | } |
29967ef6 XL |
161 | } |
162 | } | |
163 | } | |
164 | ||
5869c6ff | 165 | pub(super) fn to_casted_value<'tcx>( |
6a06907d | 166 | fx: &mut FunctionCx<'_, '_, 'tcx>, |
5869c6ff | 167 | arg: CValue<'tcx>, |
f2b60f7d | 168 | cast: &CastTarget, |
5869c6ff XL |
169 | ) -> SmallVec<[Value; 2]> { |
170 | let (ptr, meta) = arg.force_stack(fx); | |
171 | assert!(meta.is_none()); | |
172 | let mut offset = 0; | |
173 | cast_target_to_abi_params(cast) | |
174 | .into_iter() | |
175 | .map(|param| { | |
6a06907d | 176 | let val = ptr.offset_i64(fx, offset).load(fx, param.value_type, MemFlags::new()); |
5869c6ff XL |
177 | offset += i64::from(param.value_type.bytes()); |
178 | val | |
179 | }) | |
180 | .collect() | |
181 | } | |
182 | ||
183 | pub(super) fn from_casted_value<'tcx>( | |
6a06907d | 184 | fx: &mut FunctionCx<'_, '_, 'tcx>, |
5869c6ff XL |
185 | block_params: &[Value], |
186 | layout: TyAndLayout<'tcx>, | |
f2b60f7d | 187 | cast: &CastTarget, |
5869c6ff XL |
188 | ) -> CValue<'tcx> { |
189 | let abi_params = cast_target_to_abi_params(cast); | |
6a06907d | 190 | let abi_param_size: u32 = abi_params.iter().map(|param| param.value_type.bytes()).sum(); |
5869c6ff | 191 | let layout_size = u32::try_from(layout.size.bytes()).unwrap(); |
f2b60f7d | 192 | let stack_slot = fx.bcx.create_sized_stack_slot(StackSlotData { |
5869c6ff XL |
193 | kind: StackSlotKind::ExplicitSlot, |
194 | // FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to | |
195 | // specify stack slot alignment. | |
2b03887a | 196 | // Stack slot size may be bigger for example `[u8; 3]` which is packed into an `i32`. |
5869c6ff XL |
197 | // It may also be smaller for example when the type is a wrapper around an integer with a |
198 | // larger alignment than the integer. | |
199 | size: (std::cmp::max(abi_param_size, layout_size) + 15) / 16 * 16, | |
5869c6ff | 200 | }); |
f2b60f7d | 201 | let ptr = Pointer::stack_slot(stack_slot); |
5869c6ff | 202 | let mut offset = 0; |
cdc7bbd5 | 203 | let mut block_params_iter = block_params.iter().copied(); |
5869c6ff XL |
204 | for param in abi_params { |
205 | let val = ptr.offset_i64(fx, offset).store( | |
206 | fx, | |
207 | block_params_iter.next().unwrap(), | |
208 | MemFlags::new(), | |
209 | ); | |
210 | offset += i64::from(param.value_type.bytes()); | |
211 | val | |
212 | } | |
213 | assert_eq!(block_params_iter.next(), None, "Leftover block param"); | |
214 | CValue::by_ref(ptr, layout) | |
215 | } | |
216 | ||
29967ef6 XL |
217 | /// Get a set of values to be passed as function arguments. |
218 | pub(super) fn adjust_arg_for_abi<'tcx>( | |
6a06907d | 219 | fx: &mut FunctionCx<'_, '_, 'tcx>, |
29967ef6 | 220 | arg: CValue<'tcx>, |
5869c6ff | 221 | arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, |
94222f64 | 222 | is_owned: bool, |
5869c6ff | 223 | ) -> SmallVec<[Value; 2]> { |
064997fb | 224 | assert_assignable(fx, arg.layout().ty, arg_abi.layout.ty, 16); |
5869c6ff XL |
225 | match arg_abi.mode { |
226 | PassMode::Ignore => smallvec![], | |
227 | PassMode::Direct(_) => smallvec![arg.load_scalar(fx)], | |
228 | PassMode::Pair(_, _) => { | |
29967ef6 | 229 | let (a, b) = arg.load_scalar_pair(fx); |
5869c6ff | 230 | smallvec![a, b] |
29967ef6 | 231 | } |
f2b60f7d | 232 | PassMode::Cast(ref cast, _) => to_casted_value(fx, arg, cast), |
94222f64 XL |
233 | PassMode::Indirect { .. } => { |
234 | if is_owned { | |
235 | match arg.force_stack(fx) { | |
236 | (ptr, None) => smallvec![ptr.get_addr(fx)], | |
237 | (ptr, Some(meta)) => smallvec![ptr.get_addr(fx), meta], | |
238 | } | |
239 | } else { | |
240 | // Ownership of the value at the backing storage for an argument is passed to the | |
241 | // callee per the ABI, so we must make a copy of the argument unless the argument | |
242 | // local is moved. | |
243 | let place = CPlace::new_stack_slot(fx, arg.layout()); | |
244 | place.write_cvalue(fx, arg); | |
245 | smallvec![place.to_ptr().get_addr(fx)] | |
246 | } | |
247 | } | |
29967ef6 XL |
248 | } |
249 | } | |
250 | ||
251 | /// Create a [`CValue`] containing the value of a function parameter adding clif function parameters | |
252 | /// as necessary. | |
253 | pub(super) fn cvalue_for_param<'tcx>( | |
6a06907d | 254 | fx: &mut FunctionCx<'_, '_, 'tcx>, |
cdc7bbd5 XL |
255 | local: Option<mir::Local>, |
256 | local_field: Option<usize>, | |
5869c6ff XL |
257 | arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, |
258 | block_params_iter: &mut impl Iterator<Item = Value>, | |
29967ef6 | 259 | ) -> Option<CValue<'tcx>> { |
5869c6ff XL |
260 | let block_params = arg_abi |
261 | .get_abi_param(fx.tcx) | |
262 | .into_iter() | |
263 | .map(|abi_param| { | |
264 | let block_param = block_params_iter.next().unwrap(); | |
6a06907d | 265 | assert_eq!(fx.bcx.func.dfg.value_type(block_param), abi_param.value_type); |
5869c6ff XL |
266 | block_param |
267 | }) | |
268 | .collect::<SmallVec<[_; 2]>>(); | |
29967ef6 | 269 | |
29967ef6 XL |
270 | crate::abi::comments::add_arg_comment( |
271 | fx, | |
272 | "arg", | |
273 | local, | |
274 | local_field, | |
5869c6ff | 275 | &block_params, |
f2b60f7d | 276 | &arg_abi.mode, |
5869c6ff | 277 | arg_abi.layout, |
29967ef6 XL |
278 | ); |
279 | ||
5869c6ff XL |
280 | match arg_abi.mode { |
281 | PassMode::Ignore => None, | |
282 | PassMode::Direct(_) => { | |
283 | assert_eq!(block_params.len(), 1, "{:?}", block_params); | |
284 | Some(CValue::by_val(block_params[0], arg_abi.layout)) | |
285 | } | |
286 | PassMode::Pair(_, _) => { | |
287 | assert_eq!(block_params.len(), 2, "{:?}", block_params); | |
6a06907d | 288 | Some(CValue::by_val_pair(block_params[0], block_params[1], arg_abi.layout)) |
5869c6ff | 289 | } |
f2b60f7d FG |
290 | PassMode::Cast(ref cast, _) => { |
291 | Some(from_casted_value(fx, &block_params, arg_abi.layout, cast)) | |
292 | } | |
6a06907d | 293 | PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } => { |
5869c6ff | 294 | assert_eq!(block_params.len(), 1, "{:?}", block_params); |
6a06907d | 295 | Some(CValue::by_ref(Pointer::new(block_params[0]), arg_abi.layout)) |
29967ef6 | 296 | } |
6a06907d | 297 | PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => { |
5869c6ff XL |
298 | assert_eq!(block_params.len(), 2, "{:?}", block_params); |
299 | Some(CValue::by_ref_unsized( | |
300 | Pointer::new(block_params[0]), | |
301 | block_params[1], | |
302 | arg_abi.layout, | |
303 | )) | |
29967ef6 XL |
304 | } |
305 | } | |
306 | } |