]>
Commit | Line | Data |
---|---|---|
ba9703b0 | 1 | use crate::abi::{self, Abi, Align, FieldsShape, Size}; |
94222f64 | 2 | use crate::abi::{HasDataLayout, TyAbiInterface, TyAndLayout}; |
9fa01778 | 3 | use crate::spec::{self, HasTargetSpec}; |
5099ac24 | 4 | use rustc_span::Symbol; |
c295e0f8 | 5 | use std::fmt; |
83c7162d XL |
6 | |
7 | mod aarch64; | |
b7449926 | 8 | mod amdgpu; |
83c7162d | 9 | mod arm; |
f035d41b | 10 | mod avr; |
17df50a5 | 11 | mod bpf; |
83c7162d | 12 | mod hexagon; |
c295e0f8 | 13 | mod m68k; |
83c7162d XL |
14 | mod mips; |
15 | mod mips64; | |
16 | mod msp430; | |
83c7162d XL |
17 | mod nvptx64; |
18 | mod powerpc; | |
19 | mod powerpc64; | |
b7449926 | 20 | mod riscv; |
83c7162d XL |
21 | mod s390x; |
22 | mod sparc; | |
23 | mod sparc64; | |
cdc7bbd5 | 24 | mod wasm; |
83c7162d XL |
25 | mod x86; |
26 | mod x86_64; | |
27 | mod x86_win64; | |
532ac7d7 | 28 | |
f2b60f7d | 29 | #[derive(PartialEq, Eq, Hash, Debug, HashStable_Generic)] |
83c7162d | 30 | pub enum PassMode { |
e74abb32 | 31 | /// Ignore the argument. |
5869c6ff XL |
32 | /// |
33 | /// The argument is either uninhabited or a ZST. | |
e74abb32 | 34 | Ignore, |
83c7162d | 35 | /// Pass the argument directly. |
5869c6ff XL |
36 | /// |
37 | /// The argument has a layout abi of `Scalar`, `Vector` or in rare cases `Aggregate`. | |
83c7162d XL |
38 | Direct(ArgAttributes), |
39 | /// Pass a pair's elements directly in two arguments. | |
5869c6ff XL |
40 | /// |
41 | /// The argument has a layout abi of `ScalarPair`. | |
83c7162d | 42 | Pair(ArgAttributes, ArgAttributes), |
f2b60f7d FG |
43 | /// Pass the argument after casting it, to either a single uniform or a |
44 | /// pair of registers. The bool indicates if a `Reg::i32()` dummy argument | |
45 | /// is emitted before the real argument. | |
46 | Cast(Box<CastTarget>, bool), | |
83c7162d | 47 | /// Pass the argument indirectly via a hidden pointer. |
fc512014 | 48 | /// The `extra_attrs` value, if any, is for the extra data (vtable or length) |
b7449926 | 49 | /// which indicates that it refers to an unsized rvalue. |
fc512014 XL |
50 | /// `on_stack` defines that the the value should be passed at a fixed |
51 | /// stack offset in accordance to the ABI rather than passed using a | |
52 | /// pointer. This corresponds to the `byval` LLVM argument attribute. | |
53 | Indirect { attrs: ArgAttributes, extra_attrs: Option<ArgAttributes>, on_stack: bool }, | |
83c7162d XL |
54 | } |
55 | ||
56 | // Hack to disable non_upper_case_globals only for the bitflags! and not for the rest | |
57 | // of this module | |
9fa01778 | 58 | pub use attr_impl::ArgAttribute; |
83c7162d XL |
59 | |
60 | #[allow(non_upper_case_globals)] | |
61 | #[allow(unused)] | |
62 | mod attr_impl { | |
63 | // The subset of llvm::Attribute needed for arguments, packed into a bitfield. | |
9fa01778 | 64 | bitflags::bitflags! { |
c295e0f8 | 65 | #[derive(Default, HashStable_Generic)] |
83c7162d | 66 | pub struct ArgAttribute: u16 { |
83c7162d XL |
67 | const NoAlias = 1 << 1; |
68 | const NoCapture = 1 << 2; | |
69 | const NonNull = 1 << 3; | |
70 | const ReadOnly = 1 << 4; | |
cdc7bbd5 | 71 | const InReg = 1 << 5; |
3c0e092e XL |
72 | // Due to past miscompiles in LLVM, we use a separate attribute for |
73 | // &mut arguments, so that the codegen backend can decide whether | |
74 | // or not to actually emit the attribute. It can also be controlled | |
75 | // with the `-Zmutable-noalias` debugging option. | |
cdc7bbd5 | 76 | const NoAliasMutRef = 1 << 6; |
5099ac24 | 77 | const NoUndef = 1 << 7; |
83c7162d XL |
78 | } |
79 | } | |
80 | } | |
81 | ||
fc512014 | 82 | /// Sometimes an ABI requires small integers to be extended to a full or partial register. This enum |
cdc7bbd5 XL |
83 | /// defines if this extension should be zero-extension or sign-extension when necessary. When it is |
84 | /// not necessary to extend the argument, this enum is ignored. | |
c295e0f8 | 85 | #[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)] |
fc512014 XL |
86 | pub enum ArgExtension { |
87 | None, | |
88 | Zext, | |
89 | Sext, | |
90 | } | |
91 | ||
83c7162d XL |
92 | /// A compact representation of LLVM attributes (at least those relevant for this module) |
93 | /// that can be manipulated without interacting with LLVM's Attribute machinery. | |
c295e0f8 | 94 | #[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)] |
83c7162d XL |
95 | pub struct ArgAttributes { |
96 | pub regular: ArgAttribute, | |
fc512014 | 97 | pub arg_ext: ArgExtension, |
60c5eb7d XL |
98 | /// The minimum size of the pointee, guaranteed to be valid for the duration of the whole call |
99 | /// (corresponding to LLVM's dereferenceable and dereferenceable_or_null attributes). | |
83c7162d | 100 | pub pointee_size: Size, |
dfeec247 | 101 | pub pointee_align: Option<Align>, |
83c7162d XL |
102 | } |
103 | ||
104 | impl ArgAttributes { | |
105 | pub fn new() -> Self { | |
106 | ArgAttributes { | |
107 | regular: ArgAttribute::default(), | |
fc512014 | 108 | arg_ext: ArgExtension::None, |
94b46f34 | 109 | pointee_size: Size::ZERO, |
83c7162d XL |
110 | pointee_align: None, |
111 | } | |
112 | } | |
113 | ||
fc512014 | 114 | pub fn ext(&mut self, ext: ArgExtension) -> &mut Self { |
5869c6ff XL |
115 | assert!( |
116 | self.arg_ext == ArgExtension::None || self.arg_ext == ext, | |
117 | "cannot set {:?} when {:?} is already set", | |
118 | ext, | |
119 | self.arg_ext | |
120 | ); | |
fc512014 XL |
121 | self.arg_ext = ext; |
122 | self | |
123 | } | |
124 | ||
83c7162d | 125 | pub fn set(&mut self, attr: ArgAttribute) -> &mut Self { |
b7449926 | 126 | self.regular |= attr; |
83c7162d XL |
127 | self |
128 | } | |
129 | ||
130 | pub fn contains(&self, attr: ArgAttribute) -> bool { | |
131 | self.regular.contains(attr) | |
132 | } | |
133 | } | |
134 | ||
c295e0f8 | 135 | #[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)] |
83c7162d XL |
136 | pub enum RegKind { |
137 | Integer, | |
138 | Float, | |
dfeec247 | 139 | Vector, |
83c7162d XL |
140 | } |
141 | ||
c295e0f8 | 142 | #[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)] |
83c7162d XL |
143 | pub struct Reg { |
144 | pub kind: RegKind, | |
145 | pub size: Size, | |
146 | } | |
147 | ||
148 | macro_rules! reg_ctor { | |
149 | ($name:ident, $kind:ident, $bits:expr) => { | |
150 | pub fn $name() -> Reg { | |
dfeec247 | 151 | Reg { kind: RegKind::$kind, size: Size::from_bits($bits) } |
83c7162d | 152 | } |
dfeec247 | 153 | }; |
83c7162d XL |
154 | } |
155 | ||
156 | impl Reg { | |
157 | reg_ctor!(i8, Integer, 8); | |
158 | reg_ctor!(i16, Integer, 16); | |
159 | reg_ctor!(i32, Integer, 32); | |
160 | reg_ctor!(i64, Integer, 64); | |
74b04a01 | 161 | reg_ctor!(i128, Integer, 128); |
83c7162d XL |
162 | |
163 | reg_ctor!(f32, Float, 32); | |
164 | reg_ctor!(f64, Float, 64); | |
165 | } | |
166 | ||
167 | impl Reg { | |
a1dfa0c6 | 168 | pub fn align<C: HasDataLayout>(&self, cx: &C) -> Align { |
83c7162d XL |
169 | let dl = cx.data_layout(); |
170 | match self.kind { | |
dfeec247 XL |
171 | RegKind::Integer => match self.size.bits() { |
172 | 1 => dl.i1_align.abi, | |
173 | 2..=8 => dl.i8_align.abi, | |
174 | 9..=16 => dl.i16_align.abi, | |
175 | 17..=32 => dl.i32_align.abi, | |
176 | 33..=64 => dl.i64_align.abi, | |
177 | 65..=128 => dl.i128_align.abi, | |
178 | _ => panic!("unsupported integer: {:?}", self), | |
179 | }, | |
180 | RegKind::Float => match self.size.bits() { | |
181 | 32 => dl.f32_align.abi, | |
182 | 64 => dl.f64_align.abi, | |
183 | _ => panic!("unsupported float: {:?}", self), | |
184 | }, | |
a1dfa0c6 | 185 | RegKind::Vector => dl.vector_align(self.size).abi, |
83c7162d XL |
186 | } |
187 | } | |
188 | } | |
189 | ||
190 | /// An argument passed entirely registers with the | |
0731742a | 191 | /// same kind (e.g., HFA / HVA on PPC64 and AArch64). |
c295e0f8 | 192 | #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, HashStable_Generic)] |
83c7162d XL |
193 | pub struct Uniform { |
194 | pub unit: Reg, | |
195 | ||
196 | /// The total size of the argument, which can be: | |
9fa01778 XL |
197 | /// * equal to `unit.size` (one scalar/vector), |
198 | /// * a multiple of `unit.size` (an array of scalar/vectors), | |
83c7162d | 199 | /// * if `unit.kind` is `Integer`, the last element |
0731742a | 200 | /// can be shorter, i.e., `{ i64, i64, i32 }` for |
9fa01778 | 201 | /// 64-bit integers with a total size of 20 bytes. |
83c7162d XL |
202 | pub total: Size, |
203 | } | |
204 | ||
205 | impl From<Reg> for Uniform { | |
206 | fn from(unit: Reg) -> Uniform { | |
dfeec247 | 207 | Uniform { unit, total: unit.size } |
83c7162d XL |
208 | } |
209 | } | |
210 | ||
211 | impl Uniform { | |
a1dfa0c6 | 212 | pub fn align<C: HasDataLayout>(&self, cx: &C) -> Align { |
83c7162d XL |
213 | self.unit.align(cx) |
214 | } | |
215 | } | |
216 | ||
c295e0f8 | 217 | #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, HashStable_Generic)] |
83c7162d | 218 | pub struct CastTarget { |
a2a8927a | 219 | pub prefix: [Option<Reg>; 8], |
83c7162d | 220 | pub rest: Uniform, |
a2a8927a | 221 | pub attrs: ArgAttributes, |
83c7162d XL |
222 | } |
223 | ||
224 | impl From<Reg> for CastTarget { | |
225 | fn from(unit: Reg) -> CastTarget { | |
226 | CastTarget::from(Uniform::from(unit)) | |
227 | } | |
228 | } | |
229 | ||
230 | impl From<Uniform> for CastTarget { | |
231 | fn from(uniform: Uniform) -> CastTarget { | |
a2a8927a XL |
232 | CastTarget { |
233 | prefix: [None; 8], | |
234 | rest: uniform, | |
235 | attrs: ArgAttributes { | |
236 | regular: ArgAttribute::default(), | |
237 | arg_ext: ArgExtension::None, | |
238 | pointee_size: Size::ZERO, | |
239 | pointee_align: None, | |
240 | }, | |
241 | } | |
83c7162d XL |
242 | } |
243 | } | |
244 | ||
245 | impl CastTarget { | |
246 | pub fn pair(a: Reg, b: Reg) -> CastTarget { | |
247 | CastTarget { | |
a2a8927a | 248 | prefix: [Some(a), None, None, None, None, None, None, None], |
dfeec247 | 249 | rest: Uniform::from(b), |
a2a8927a XL |
250 | attrs: ArgAttributes { |
251 | regular: ArgAttribute::default(), | |
252 | arg_ext: ArgExtension::None, | |
253 | pointee_size: Size::ZERO, | |
254 | pointee_align: None, | |
255 | }, | |
83c7162d XL |
256 | } |
257 | } | |
258 | ||
a2a8927a XL |
259 | pub fn size<C: HasDataLayout>(&self, _cx: &C) -> Size { |
260 | let mut size = self.rest.total; | |
261 | for i in 0..self.prefix.iter().count() { | |
262 | match self.prefix[i] { | |
263 | Some(v) => size += Size { raw: v.size.bytes() }, | |
264 | None => {} | |
265 | } | |
266 | } | |
267 | return size; | |
83c7162d XL |
268 | } |
269 | ||
a1dfa0c6 | 270 | pub fn align<C: HasDataLayout>(&self, cx: &C) -> Align { |
dfeec247 XL |
271 | self.prefix |
272 | .iter() | |
a2a8927a | 273 | .filter_map(|x| x.map(|reg| reg.align(cx))) |
dfeec247 XL |
274 | .fold(cx.data_layout().aggregate_align.abi.max(self.rest.align(cx)), |acc, align| { |
275 | acc.max(align) | |
276 | }) | |
83c7162d XL |
277 | } |
278 | } | |
279 | ||
74b04a01 | 280 | /// Return value from the `homogeneous_aggregate` test function. |
9fa01778 XL |
281 | #[derive(Copy, Clone, Debug)] |
282 | pub enum HomogeneousAggregate { | |
283 | /// Yes, all the "leaf fields" of this struct are passed in the | |
284 | /// same way (specified in the `Reg` value). | |
285 | Homogeneous(Reg), | |
286 | ||
9fa01778 XL |
287 | /// There are no leaf fields at all. |
288 | NoData, | |
289 | } | |
290 | ||
74b04a01 XL |
291 | /// Error from the `homogeneous_aggregate` test function, indicating |
292 | /// there are distinct leaf fields passed in different ways, | |
293 | /// or this is uninhabited. | |
294 | #[derive(Copy, Clone, Debug)] | |
295 | pub struct Heterogeneous; | |
296 | ||
9fa01778 XL |
297 | impl HomogeneousAggregate { |
298 | /// If this is a homogeneous aggregate, returns the homogeneous | |
299 | /// unit, else `None`. | |
300 | pub fn unit(self) -> Option<Reg> { | |
74b04a01 XL |
301 | match self { |
302 | HomogeneousAggregate::Homogeneous(reg) => Some(reg), | |
303 | HomogeneousAggregate::NoData => None, | |
304 | } | |
305 | } | |
306 | ||
307 | /// Try to combine two `HomogeneousAggregate`s, e.g. from two fields in | |
308 | /// the same `struct`. Only succeeds if only one of them has any data, | |
309 | /// or both units are identical. | |
310 | fn merge(self, other: HomogeneousAggregate) -> Result<HomogeneousAggregate, Heterogeneous> { | |
311 | match (self, other) { | |
312 | (x, HomogeneousAggregate::NoData) | (HomogeneousAggregate::NoData, x) => Ok(x), | |
313 | ||
314 | (HomogeneousAggregate::Homogeneous(a), HomogeneousAggregate::Homogeneous(b)) => { | |
315 | if a != b { | |
316 | return Err(Heterogeneous); | |
317 | } | |
318 | Ok(self) | |
319 | } | |
320 | } | |
9fa01778 XL |
321 | } |
322 | } | |
323 | ||
ba9703b0 | 324 | impl<'a, Ty> TyAndLayout<'a, Ty> { |
83c7162d XL |
325 | fn is_aggregate(&self) -> bool { |
326 | match self.abi { | |
dfeec247 XL |
327 | Abi::Uninhabited | Abi::Scalar(_) | Abi::Vector { .. } => false, |
328 | Abi::ScalarPair(..) | Abi::Aggregate { .. } => true, | |
83c7162d XL |
329 | } |
330 | } | |
331 | ||
74b04a01 XL |
332 | /// Returns `Homogeneous` if this layout is an aggregate containing fields of |
333 | /// only a single type (e.g., `(u32, u32)`). Such aggregates are often | |
9fa01778 XL |
334 | /// special-cased in ABIs. |
335 | /// | |
336 | /// Note: We generally ignore fields of zero-sized type when computing | |
337 | /// this value (see #56877). | |
338 | /// | |
339 | /// This is public so that it can be used in unit tests, but | |
340 | /// should generally only be relevant to the ABI details of | |
341 | /// specific targets. | |
74b04a01 | 342 | pub fn homogeneous_aggregate<C>(&self, cx: &C) -> Result<HomogeneousAggregate, Heterogeneous> |
dfeec247 | 343 | where |
94222f64 | 344 | Ty: TyAbiInterface<'a, C> + Copy, |
83c7162d XL |
345 | { |
346 | match self.abi { | |
74b04a01 | 347 | Abi::Uninhabited => Err(Heterogeneous), |
83c7162d XL |
348 | |
349 | // The primitive for this algorithm. | |
c295e0f8 | 350 | Abi::Scalar(scalar) => { |
04454e1e | 351 | let kind = match scalar.primitive() { |
dfeec247 | 352 | abi::Int(..) | abi::Pointer => RegKind::Integer, |
60c5eb7d | 353 | abi::F32 | abi::F64 => RegKind::Float, |
83c7162d | 354 | }; |
74b04a01 | 355 | Ok(HomogeneousAggregate::Homogeneous(Reg { kind, size: self.size })) |
83c7162d XL |
356 | } |
357 | ||
358 | Abi::Vector { .. } => { | |
9fa01778 | 359 | assert!(!self.is_zst()); |
74b04a01 XL |
360 | Ok(HomogeneousAggregate::Homogeneous(Reg { |
361 | kind: RegKind::Vector, | |
362 | size: self.size, | |
363 | })) | |
83c7162d XL |
364 | } |
365 | ||
dfeec247 | 366 | Abi::ScalarPair(..) | Abi::Aggregate { .. } => { |
ba9703b0 | 367 | // Helper for computing `homogeneous_aggregate`, allowing a custom |
74b04a01 XL |
368 | // starting offset (used below for handling variants). |
369 | let from_fields_at = | |
370 | |layout: Self, | |
371 | start: Size| | |
372 | -> Result<(HomogeneousAggregate, Size), Heterogeneous> { | |
373 | let is_union = match layout.fields { | |
ba9703b0 XL |
374 | FieldsShape::Primitive => { |
375 | unreachable!("aggregates can't have `FieldsShape::Primitive`") | |
376 | } | |
377 | FieldsShape::Array { count, .. } => { | |
74b04a01 XL |
378 | assert_eq!(start, Size::ZERO); |
379 | ||
380 | let result = if count > 0 { | |
381 | layout.field(cx, 0).homogeneous_aggregate(cx)? | |
382 | } else { | |
383 | HomogeneousAggregate::NoData | |
384 | }; | |
385 | return Ok((result, layout.size)); | |
386 | } | |
ba9703b0 XL |
387 | FieldsShape::Union(_) => true, |
388 | FieldsShape::Arbitrary { .. } => false, | |
74b04a01 | 389 | }; |
83c7162d | 390 | |
74b04a01 XL |
391 | let mut result = HomogeneousAggregate::NoData; |
392 | let mut total = start; | |
83c7162d | 393 | |
74b04a01 XL |
394 | for i in 0..layout.fields.count() { |
395 | if !is_union && total != layout.fields.offset(i) { | |
396 | return Err(Heterogeneous); | |
397 | } | |
9fa01778 | 398 | |
74b04a01 XL |
399 | let field = layout.field(cx, i); |
400 | ||
401 | result = result.merge(field.homogeneous_aggregate(cx)?)?; | |
402 | ||
403 | // Keep track of the offset (without padding). | |
404 | let size = field.size; | |
405 | if is_union { | |
406 | total = total.max(size); | |
407 | } else { | |
408 | total += size; | |
83c7162d XL |
409 | } |
410 | } | |
83c7162d | 411 | |
74b04a01 XL |
412 | Ok((result, total)) |
413 | }; | |
414 | ||
415 | let (mut result, mut total) = from_fields_at(*self, Size::ZERO)?; | |
416 | ||
417 | match &self.variants { | |
418 | abi::Variants::Single { .. } => {} | |
419 | abi::Variants::Multiple { variants, .. } => { | |
420 | // Treat enum variants like union members. | |
421 | // HACK(eddyb) pretend the `enum` field (discriminant) | |
422 | // is at the start of every variant (otherwise the gap | |
423 | // at the start of all variants would disqualify them). | |
424 | // | |
425 | // NB: for all tagged `enum`s (which include all non-C-like | |
426 | // `enum`s with defined FFI representation), this will | |
427 | // match the homogeneous computation on the equivalent | |
428 | // `struct { tag; union { variant1; ... } }` and/or | |
429 | // `union { struct { tag; variant1; } ... }` | |
430 | // (the offsets of variant fields should be identical | |
431 | // between the two for either to be a homogeneous aggregate). | |
432 | let variant_start = total; | |
433 | for variant_idx in variants.indices() { | |
434 | let (variant_result, variant_total) = | |
435 | from_fields_at(self.for_variant(cx, variant_idx), variant_start)?; | |
436 | ||
437 | result = result.merge(variant_result)?; | |
438 | total = total.max(variant_total); | |
439 | } | |
83c7162d XL |
440 | } |
441 | } | |
442 | ||
443 | // There needs to be no padding. | |
444 | if total != self.size { | |
74b04a01 | 445 | Err(Heterogeneous) |
83c7162d | 446 | } else { |
9fa01778 | 447 | match result { |
74b04a01 | 448 | HomogeneousAggregate::Homogeneous(_) => { |
9fa01778 | 449 | assert_ne!(total, Size::ZERO); |
9fa01778 | 450 | } |
74b04a01 | 451 | HomogeneousAggregate::NoData => { |
9fa01778 | 452 | assert_eq!(total, Size::ZERO); |
9fa01778 XL |
453 | } |
454 | } | |
74b04a01 | 455 | Ok(result) |
83c7162d XL |
456 | } |
457 | } | |
458 | } | |
459 | } | |
460 | } | |
461 | ||
462 | /// Information about how to pass an argument to, | |
463 | /// or return a value from, a function, under some ABI. | |
c295e0f8 | 464 | #[derive(PartialEq, Eq, Hash, Debug, HashStable_Generic)] |
60c5eb7d | 465 | pub struct ArgAbi<'a, Ty> { |
ba9703b0 | 466 | pub layout: TyAndLayout<'a, Ty>, |
83c7162d XL |
467 | pub mode: PassMode, |
468 | } | |
469 | ||
60c5eb7d | 470 | impl<'a, Ty> ArgAbi<'a, Ty> { |
5869c6ff XL |
471 | pub fn new( |
472 | cx: &impl HasDataLayout, | |
473 | layout: TyAndLayout<'a, Ty>, | |
c295e0f8 | 474 | scalar_attrs: impl Fn(&TyAndLayout<'a, Ty>, abi::Scalar, Size) -> ArgAttributes, |
5869c6ff | 475 | ) -> Self { |
c295e0f8 | 476 | let mode = match layout.abi { |
5869c6ff XL |
477 | Abi::Uninhabited => PassMode::Ignore, |
478 | Abi::Scalar(scalar) => PassMode::Direct(scalar_attrs(&layout, scalar, Size::ZERO)), | |
479 | Abi::ScalarPair(a, b) => PassMode::Pair( | |
480 | scalar_attrs(&layout, a, Size::ZERO), | |
04454e1e | 481 | scalar_attrs(&layout, b, a.size(cx).align_to(b.align(cx).abi)), |
5869c6ff XL |
482 | ), |
483 | Abi::Vector { .. } => PassMode::Direct(ArgAttributes::new()), | |
484 | Abi::Aggregate { .. } => PassMode::Direct(ArgAttributes::new()), | |
485 | }; | |
f2b60f7d | 486 | ArgAbi { layout, mode } |
83c7162d XL |
487 | } |
488 | ||
5869c6ff | 489 | fn indirect_pass_mode(layout: &TyAndLayout<'a, Ty>) -> PassMode { |
83c7162d XL |
490 | let mut attrs = ArgAttributes::new(); |
491 | ||
492 | // For non-immediate arguments the callee gets its own copy of | |
493 | // the value on the stack, so there are no aliases. It's also | |
494 | // program-invisible so can't possibly capture | |
5099ac24 FG |
495 | attrs |
496 | .set(ArgAttribute::NoAlias) | |
497 | .set(ArgAttribute::NoCapture) | |
498 | .set(ArgAttribute::NonNull) | |
499 | .set(ArgAttribute::NoUndef); | |
5869c6ff | 500 | attrs.pointee_size = layout.size; |
83c7162d XL |
501 | // FIXME(eddyb) We should be doing this, but at least on |
502 | // i686-pc-windows-msvc, it results in wrong stack offsets. | |
5869c6ff | 503 | // attrs.pointee_align = Some(layout.align.abi); |
83c7162d | 504 | |
5869c6ff | 505 | let extra_attrs = layout.is_unsized().then_some(ArgAttributes::new()); |
b7449926 | 506 | |
5869c6ff XL |
507 | PassMode::Indirect { attrs, extra_attrs, on_stack: false } |
508 | } | |
509 | ||
510 | pub fn make_indirect(&mut self) { | |
511 | match self.mode { | |
512 | PassMode::Direct(_) | PassMode::Pair(_, _) => {} | |
513 | PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: false } => return, | |
514 | _ => panic!("Tried to make {:?} indirect", self.mode), | |
515 | } | |
516 | ||
517 | self.mode = Self::indirect_pass_mode(&self.layout); | |
83c7162d XL |
518 | } |
519 | ||
520 | pub fn make_indirect_byval(&mut self) { | |
521 | self.make_indirect(); | |
522 | match self.mode { | |
fc512014 XL |
523 | PassMode::Indirect { attrs: _, extra_attrs: _, ref mut on_stack } => { |
524 | *on_stack = true; | |
83c7162d | 525 | } |
dfeec247 | 526 | _ => unreachable!(), |
83c7162d XL |
527 | } |
528 | } | |
529 | ||
530 | pub fn extend_integer_width_to(&mut self, bits: u64) { | |
531 | // Only integers have signedness | |
c295e0f8 | 532 | if let Abi::Scalar(scalar) = self.layout.abi { |
04454e1e | 533 | if let abi::Int(i, signed) = scalar.primitive() { |
83c7162d XL |
534 | if i.size().bits() < bits { |
535 | if let PassMode::Direct(ref mut attrs) = self.mode { | |
fc512014 XL |
536 | if signed { |
537 | attrs.ext(ArgExtension::Sext) | |
538 | } else { | |
539 | attrs.ext(ArgExtension::Zext) | |
540 | }; | |
83c7162d XL |
541 | } |
542 | } | |
543 | } | |
544 | } | |
545 | } | |
546 | ||
547 | pub fn cast_to<T: Into<CastTarget>>(&mut self, target: T) { | |
f2b60f7d | 548 | self.mode = PassMode::Cast(Box::new(target.into()), false); |
83c7162d XL |
549 | } |
550 | ||
f2b60f7d FG |
551 | pub fn cast_to_and_pad_i32<T: Into<CastTarget>>(&mut self, target: T, pad_i32: bool) { |
552 | self.mode = PassMode::Cast(Box::new(target.into()), pad_i32); | |
83c7162d XL |
553 | } |
554 | ||
555 | pub fn is_indirect(&self) -> bool { | |
5869c6ff | 556 | matches!(self.mode, PassMode::Indirect { .. }) |
b7449926 XL |
557 | } |
558 | ||
559 | pub fn is_sized_indirect(&self) -> bool { | |
fc512014 | 560 | matches!(self.mode, PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ }) |
b7449926 XL |
561 | } |
562 | ||
563 | pub fn is_unsized_indirect(&self) -> bool { | |
fc512014 | 564 | matches!(self.mode, PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ }) |
83c7162d XL |
565 | } |
566 | ||
567 | pub fn is_ignore(&self) -> bool { | |
29967ef6 | 568 | matches!(self.mode, PassMode::Ignore) |
83c7162d XL |
569 | } |
570 | } | |
571 | ||
c295e0f8 | 572 | #[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)] |
83c7162d | 573 | pub enum Conv { |
60c5eb7d XL |
574 | // General language calling conventions, for which every target |
575 | // should have its own backend (e.g. LLVM) support. | |
83c7162d | 576 | C, |
60c5eb7d XL |
577 | Rust, |
578 | ||
923072b8 FG |
579 | /// For things unlikely to be called, where smaller caller codegen is |
580 | /// preferred over raw speed. | |
581 | /// Stronger than just `#[cold]` because `fn` pointers might be incompatible. | |
582 | RustCold, | |
583 | ||
60c5eb7d | 584 | // Target-specific calling conventions. |
83c7162d | 585 | ArmAapcs, |
5869c6ff | 586 | CCmseNonSecureCall, |
83c7162d XL |
587 | |
588 | Msp430Intr, | |
589 | ||
590 | PtxKernel, | |
591 | ||
592 | X86Fastcall, | |
593 | X86Intr, | |
594 | X86Stdcall, | |
595 | X86ThisCall, | |
596 | X86VectorCall, | |
597 | ||
598 | X86_64SysV, | |
599 | X86_64Win64, | |
8faf50e0 XL |
600 | |
601 | AmdGpuKernel, | |
f035d41b XL |
602 | AvrInterrupt, |
603 | AvrNonBlockingInterrupt, | |
83c7162d XL |
604 | } |
605 | ||
606 | /// Metadata describing how the arguments to a native function | |
607 | /// should be passed in order to respect the native ABI. | |
608 | /// | |
609 | /// I will do my best to describe this structure, but these | |
610 | /// comments are reverse-engineered and may be inaccurate. -NDM | |
c295e0f8 | 611 | #[derive(PartialEq, Eq, Hash, Debug, HashStable_Generic)] |
60c5eb7d | 612 | pub struct FnAbi<'a, Ty> { |
83c7162d | 613 | /// The LLVM types of each argument. |
f2b60f7d | 614 | pub args: Box<[ArgAbi<'a, Ty>]>, |
83c7162d XL |
615 | |
616 | /// LLVM return type. | |
60c5eb7d | 617 | pub ret: ArgAbi<'a, Ty>, |
83c7162d | 618 | |
532ac7d7 | 619 | pub c_variadic: bool, |
83c7162d | 620 | |
74b04a01 XL |
621 | /// The count of non-variadic arguments. |
622 | /// | |
623 | /// Should only be different from args.len() when c_variadic is true. | |
624 | /// This can be used to know whether an argument is variadic or not. | |
f2b60f7d | 625 | pub fixed_count: u32, |
74b04a01 | 626 | |
83c7162d | 627 | pub conv: Conv, |
ba9703b0 XL |
628 | |
629 | pub can_unwind: bool, | |
83c7162d XL |
630 | } |
631 | ||
c295e0f8 | 632 | /// Error produced by attempting to adjust a `FnAbi`, for a "foreign" ABI. |
5099ac24 | 633 | #[derive(Copy, Clone, Debug, HashStable_Generic)] |
c295e0f8 XL |
634 | pub enum AdjustForForeignAbiError { |
635 | /// Target architecture doesn't support "foreign" (i.e. non-Rust) ABIs. | |
5099ac24 | 636 | Unsupported { arch: Symbol, abi: spec::abi::Abi }, |
c295e0f8 XL |
637 | } |
638 | ||
639 | impl fmt::Display for AdjustForForeignAbiError { | |
640 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { | |
641 | match self { | |
642 | Self::Unsupported { arch, abi } => { | |
643 | write!(f, "target architecture {:?} does not support `extern {}` ABI", arch, abi) | |
644 | } | |
645 | } | |
646 | } | |
647 | } | |
648 | ||
60c5eb7d | 649 | impl<'a, Ty> FnAbi<'a, Ty> { |
c295e0f8 XL |
650 | pub fn adjust_for_foreign_abi<C>( |
651 | &mut self, | |
652 | cx: &C, | |
653 | abi: spec::abi::Abi, | |
654 | ) -> Result<(), AdjustForForeignAbiError> | |
dfeec247 | 655 | where |
94222f64 XL |
656 | Ty: TyAbiInterface<'a, C> + Copy, |
657 | C: HasDataLayout + HasTargetSpec, | |
83c7162d | 658 | { |
6a06907d XL |
659 | if abi == spec::abi::Abi::X86Interrupt { |
660 | if let Some(arg) = self.args.first_mut() { | |
661 | arg.make_indirect_byval(); | |
662 | } | |
663 | return Ok(()); | |
664 | } | |
665 | ||
83c7162d XL |
666 | match &cx.target_spec().arch[..] { |
667 | "x86" => { | |
064997fb FG |
668 | let flavor = if let spec::abi::Abi::Fastcall { .. } |
669 | | spec::abi::Abi::Vectorcall { .. } = abi | |
670 | { | |
671 | x86::Flavor::FastcallOrVectorcall | |
83c7162d XL |
672 | } else { |
673 | x86::Flavor::General | |
674 | }; | |
675 | x86::compute_abi_info(cx, self, flavor); | |
dfeec247 | 676 | } |
5099ac24 FG |
677 | "x86_64" => match abi { |
678 | spec::abi::Abi::SysV64 { .. } => x86_64::compute_abi_info(cx, self), | |
679 | spec::abi::Abi::Win64 { .. } => x86_win64::compute_abi_info(self), | |
680 | _ => { | |
681 | if cx.target_spec().is_like_windows { | |
682 | x86_win64::compute_abi_info(self) | |
683 | } else { | |
684 | x86_64::compute_abi_info(cx, self) | |
685 | } | |
dfeec247 | 686 | } |
5099ac24 | 687 | }, |
f2b60f7d FG |
688 | "aarch64" => { |
689 | let param_policy = if cx.target_spec().is_like_osx { | |
690 | aarch64::ParamExtension::ExtendTo32Bits | |
691 | } else { | |
692 | aarch64::ParamExtension::NoExtension | |
693 | }; | |
694 | aarch64::compute_abi_info(cx, self, param_policy) | |
695 | } | |
b7449926 | 696 | "amdgpu" => amdgpu::compute_abi_info(cx, self), |
83c7162d | 697 | "arm" => arm::compute_abi_info(cx, self), |
f035d41b | 698 | "avr" => avr::compute_abi_info(self), |
c295e0f8 | 699 | "m68k" => m68k::compute_abi_info(self), |
83c7162d XL |
700 | "mips" => mips::compute_abi_info(cx, self), |
701 | "mips64" => mips64::compute_abi_info(cx, self), | |
60c5eb7d | 702 | "powerpc" => powerpc::compute_abi_info(self), |
83c7162d XL |
703 | "powerpc64" => powerpc64::compute_abi_info(cx, self), |
704 | "s390x" => s390x::compute_abi_info(cx, self), | |
83c7162d XL |
705 | "msp430" => msp430::compute_abi_info(self), |
706 | "sparc" => sparc::compute_abi_info(cx, self), | |
707 | "sparc64" => sparc64::compute_abi_info(cx, self), | |
04454e1e FG |
708 | "nvptx64" => { |
709 | if cx.target_spec().adjust_abi(abi) == spec::abi::Abi::PtxKernel { | |
710 | nvptx64::compute_ptx_kernel_abi_info(cx, self) | |
711 | } else { | |
712 | nvptx64::compute_abi_info(self) | |
713 | } | |
714 | } | |
83c7162d | 715 | "hexagon" => hexagon::compute_abi_info(self), |
74b04a01 | 716 | "riscv32" | "riscv64" => riscv::compute_abi_info(cx, self), |
cdc7bbd5 XL |
717 | "wasm32" | "wasm64" => { |
718 | if cx.target_spec().adjust_abi(abi) == spec::abi::Abi::Wasm { | |
719 | wasm::compute_wasm_abi_info(self) | |
720 | } else { | |
721 | wasm::compute_c_abi_info(cx, self) | |
722 | } | |
723 | } | |
724 | "asmjs" => wasm::compute_c_abi_info(cx, self), | |
17df50a5 | 725 | "bpf" => bpf::compute_abi_info(self), |
c295e0f8 | 726 | arch => { |
5099ac24 FG |
727 | return Err(AdjustForForeignAbiError::Unsupported { |
728 | arch: Symbol::intern(arch), | |
729 | abi, | |
730 | }); | |
c295e0f8 | 731 | } |
83c7162d XL |
732 | } |
733 | ||
83c7162d XL |
734 | Ok(()) |
735 | } | |
736 | } | |
f2b60f7d FG |
737 | |
738 | // Some types are used a lot. Make sure they don't unintentionally get bigger. | |
739 | #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))] | |
740 | mod size_asserts { | |
741 | use super::*; | |
742 | use rustc_data_structures::static_assert_size; | |
743 | // These are in alphabetical order, which is easy to maintain. | |
744 | static_assert_size!(ArgAbi<'_, usize>, 56); | |
745 | static_assert_size!(FnAbi<'_, usize>, 80); | |
746 | } |