]>
Commit | Line | Data |
---|---|---|
ba9703b0 | 1 | use crate::abi::{self, Abi, Align, FieldsShape, Size}; |
94222f64 | 2 | use crate::abi::{HasDataLayout, TyAbiInterface, TyAndLayout}; |
9fa01778 | 3 | use crate::spec::{self, HasTargetSpec}; |
5099ac24 | 4 | use rustc_span::Symbol; |
c295e0f8 | 5 | use std::fmt; |
487cf647 | 6 | use std::str::FromStr; |
83c7162d XL |
7 | |
8 | mod aarch64; | |
b7449926 | 9 | mod amdgpu; |
83c7162d | 10 | mod arm; |
f035d41b | 11 | mod avr; |
17df50a5 | 12 | mod bpf; |
83c7162d | 13 | mod hexagon; |
487cf647 | 14 | mod loongarch; |
c295e0f8 | 15 | mod m68k; |
83c7162d XL |
16 | mod mips; |
17 | mod mips64; | |
18 | mod msp430; | |
83c7162d XL |
19 | mod nvptx64; |
20 | mod powerpc; | |
21 | mod powerpc64; | |
b7449926 | 22 | mod riscv; |
83c7162d XL |
23 | mod s390x; |
24 | mod sparc; | |
25 | mod sparc64; | |
cdc7bbd5 | 26 | mod wasm; |
83c7162d XL |
27 | mod x86; |
28 | mod x86_64; | |
29 | mod x86_win64; | |
532ac7d7 | 30 | |
f2b60f7d | 31 | #[derive(PartialEq, Eq, Hash, Debug, HashStable_Generic)] |
83c7162d | 32 | pub enum PassMode { |
e74abb32 | 33 | /// Ignore the argument. |
5869c6ff XL |
34 | /// |
35 | /// The argument is either uninhabited or a ZST. | |
e74abb32 | 36 | Ignore, |
83c7162d | 37 | /// Pass the argument directly. |
5869c6ff XL |
38 | /// |
39 | /// The argument has a layout abi of `Scalar`, `Vector` or in rare cases `Aggregate`. | |
83c7162d XL |
40 | Direct(ArgAttributes), |
41 | /// Pass a pair's elements directly in two arguments. | |
5869c6ff XL |
42 | /// |
43 | /// The argument has a layout abi of `ScalarPair`. | |
83c7162d | 44 | Pair(ArgAttributes, ArgAttributes), |
f2b60f7d FG |
45 | /// Pass the argument after casting it, to either a single uniform or a |
46 | /// pair of registers. The bool indicates if a `Reg::i32()` dummy argument | |
47 | /// is emitted before the real argument. | |
48 | Cast(Box<CastTarget>, bool), | |
83c7162d | 49 | /// Pass the argument indirectly via a hidden pointer. |
fc512014 | 50 | /// The `extra_attrs` value, if any, is for the extra data (vtable or length) |
b7449926 | 51 | /// which indicates that it refers to an unsized rvalue. |
2b03887a | 52 | /// `on_stack` defines that the value should be passed at a fixed |
fc512014 XL |
53 | /// stack offset in accordance to the ABI rather than passed using a |
54 | /// pointer. This corresponds to the `byval` LLVM argument attribute. | |
55 | Indirect { attrs: ArgAttributes, extra_attrs: Option<ArgAttributes>, on_stack: bool }, | |
83c7162d XL |
56 | } |
57 | ||
58 | // Hack to disable non_upper_case_globals only for the bitflags! and not for the rest | |
59 | // of this module | |
9fa01778 | 60 | pub use attr_impl::ArgAttribute; |
83c7162d XL |
61 | |
62 | #[allow(non_upper_case_globals)] | |
63 | #[allow(unused)] | |
64 | mod attr_impl { | |
65 | // The subset of llvm::Attribute needed for arguments, packed into a bitfield. | |
9fa01778 | 66 | bitflags::bitflags! { |
c295e0f8 | 67 | #[derive(Default, HashStable_Generic)] |
83c7162d | 68 | pub struct ArgAttribute: u16 { |
83c7162d XL |
69 | const NoAlias = 1 << 1; |
70 | const NoCapture = 1 << 2; | |
71 | const NonNull = 1 << 3; | |
72 | const ReadOnly = 1 << 4; | |
cdc7bbd5 | 73 | const InReg = 1 << 5; |
9c376795 | 74 | const NoUndef = 1 << 6; |
83c7162d XL |
75 | } |
76 | } | |
77 | } | |
78 | ||
fc512014 | 79 | /// Sometimes an ABI requires small integers to be extended to a full or partial register. This enum |
cdc7bbd5 XL |
80 | /// defines if this extension should be zero-extension or sign-extension when necessary. When it is |
81 | /// not necessary to extend the argument, this enum is ignored. | |
c295e0f8 | 82 | #[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)] |
fc512014 XL |
83 | pub enum ArgExtension { |
84 | None, | |
85 | Zext, | |
86 | Sext, | |
87 | } | |
88 | ||
83c7162d XL |
89 | /// A compact representation of LLVM attributes (at least those relevant for this module) |
90 | /// that can be manipulated without interacting with LLVM's Attribute machinery. | |
c295e0f8 | 91 | #[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)] |
83c7162d XL |
92 | pub struct ArgAttributes { |
93 | pub regular: ArgAttribute, | |
fc512014 | 94 | pub arg_ext: ArgExtension, |
60c5eb7d XL |
95 | /// The minimum size of the pointee, guaranteed to be valid for the duration of the whole call |
96 | /// (corresponding to LLVM's dereferenceable and dereferenceable_or_null attributes). | |
83c7162d | 97 | pub pointee_size: Size, |
dfeec247 | 98 | pub pointee_align: Option<Align>, |
83c7162d XL |
99 | } |
100 | ||
101 | impl ArgAttributes { | |
102 | pub fn new() -> Self { | |
103 | ArgAttributes { | |
104 | regular: ArgAttribute::default(), | |
fc512014 | 105 | arg_ext: ArgExtension::None, |
94b46f34 | 106 | pointee_size: Size::ZERO, |
83c7162d XL |
107 | pointee_align: None, |
108 | } | |
109 | } | |
110 | ||
fc512014 | 111 | pub fn ext(&mut self, ext: ArgExtension) -> &mut Self { |
5869c6ff XL |
112 | assert!( |
113 | self.arg_ext == ArgExtension::None || self.arg_ext == ext, | |
114 | "cannot set {:?} when {:?} is already set", | |
115 | ext, | |
116 | self.arg_ext | |
117 | ); | |
fc512014 XL |
118 | self.arg_ext = ext; |
119 | self | |
120 | } | |
121 | ||
83c7162d | 122 | pub fn set(&mut self, attr: ArgAttribute) -> &mut Self { |
b7449926 | 123 | self.regular |= attr; |
83c7162d XL |
124 | self |
125 | } | |
126 | ||
127 | pub fn contains(&self, attr: ArgAttribute) -> bool { | |
128 | self.regular.contains(attr) | |
129 | } | |
130 | } | |
131 | ||
c295e0f8 | 132 | #[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)] |
83c7162d XL |
133 | pub enum RegKind { |
134 | Integer, | |
135 | Float, | |
dfeec247 | 136 | Vector, |
83c7162d XL |
137 | } |
138 | ||
c295e0f8 | 139 | #[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)] |
83c7162d XL |
140 | pub struct Reg { |
141 | pub kind: RegKind, | |
142 | pub size: Size, | |
143 | } | |
144 | ||
145 | macro_rules! reg_ctor { | |
146 | ($name:ident, $kind:ident, $bits:expr) => { | |
147 | pub fn $name() -> Reg { | |
dfeec247 | 148 | Reg { kind: RegKind::$kind, size: Size::from_bits($bits) } |
83c7162d | 149 | } |
dfeec247 | 150 | }; |
83c7162d XL |
151 | } |
152 | ||
153 | impl Reg { | |
154 | reg_ctor!(i8, Integer, 8); | |
155 | reg_ctor!(i16, Integer, 16); | |
156 | reg_ctor!(i32, Integer, 32); | |
157 | reg_ctor!(i64, Integer, 64); | |
74b04a01 | 158 | reg_ctor!(i128, Integer, 128); |
83c7162d XL |
159 | |
160 | reg_ctor!(f32, Float, 32); | |
161 | reg_ctor!(f64, Float, 64); | |
162 | } | |
163 | ||
164 | impl Reg { | |
a1dfa0c6 | 165 | pub fn align<C: HasDataLayout>(&self, cx: &C) -> Align { |
83c7162d XL |
166 | let dl = cx.data_layout(); |
167 | match self.kind { | |
dfeec247 XL |
168 | RegKind::Integer => match self.size.bits() { |
169 | 1 => dl.i1_align.abi, | |
170 | 2..=8 => dl.i8_align.abi, | |
171 | 9..=16 => dl.i16_align.abi, | |
172 | 17..=32 => dl.i32_align.abi, | |
173 | 33..=64 => dl.i64_align.abi, | |
174 | 65..=128 => dl.i128_align.abi, | |
9c376795 | 175 | _ => panic!("unsupported integer: {self:?}"), |
dfeec247 XL |
176 | }, |
177 | RegKind::Float => match self.size.bits() { | |
178 | 32 => dl.f32_align.abi, | |
179 | 64 => dl.f64_align.abi, | |
9c376795 | 180 | _ => panic!("unsupported float: {self:?}"), |
dfeec247 | 181 | }, |
a1dfa0c6 | 182 | RegKind::Vector => dl.vector_align(self.size).abi, |
83c7162d XL |
183 | } |
184 | } | |
185 | } | |
186 | ||
187 | /// An argument passed entirely registers with the | |
0731742a | 188 | /// same kind (e.g., HFA / HVA on PPC64 and AArch64). |
c295e0f8 | 189 | #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, HashStable_Generic)] |
83c7162d XL |
190 | pub struct Uniform { |
191 | pub unit: Reg, | |
192 | ||
193 | /// The total size of the argument, which can be: | |
9fa01778 XL |
194 | /// * equal to `unit.size` (one scalar/vector), |
195 | /// * a multiple of `unit.size` (an array of scalar/vectors), | |
83c7162d | 196 | /// * if `unit.kind` is `Integer`, the last element |
0731742a | 197 | /// can be shorter, i.e., `{ i64, i64, i32 }` for |
9fa01778 | 198 | /// 64-bit integers with a total size of 20 bytes. |
83c7162d XL |
199 | pub total: Size, |
200 | } | |
201 | ||
202 | impl From<Reg> for Uniform { | |
203 | fn from(unit: Reg) -> Uniform { | |
dfeec247 | 204 | Uniform { unit, total: unit.size } |
83c7162d XL |
205 | } |
206 | } | |
207 | ||
208 | impl Uniform { | |
a1dfa0c6 | 209 | pub fn align<C: HasDataLayout>(&self, cx: &C) -> Align { |
83c7162d XL |
210 | self.unit.align(cx) |
211 | } | |
212 | } | |
213 | ||
c295e0f8 | 214 | #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, HashStable_Generic)] |
83c7162d | 215 | pub struct CastTarget { |
a2a8927a | 216 | pub prefix: [Option<Reg>; 8], |
83c7162d | 217 | pub rest: Uniform, |
a2a8927a | 218 | pub attrs: ArgAttributes, |
83c7162d XL |
219 | } |
220 | ||
221 | impl From<Reg> for CastTarget { | |
222 | fn from(unit: Reg) -> CastTarget { | |
223 | CastTarget::from(Uniform::from(unit)) | |
224 | } | |
225 | } | |
226 | ||
227 | impl From<Uniform> for CastTarget { | |
228 | fn from(uniform: Uniform) -> CastTarget { | |
a2a8927a XL |
229 | CastTarget { |
230 | prefix: [None; 8], | |
231 | rest: uniform, | |
232 | attrs: ArgAttributes { | |
233 | regular: ArgAttribute::default(), | |
234 | arg_ext: ArgExtension::None, | |
235 | pointee_size: Size::ZERO, | |
236 | pointee_align: None, | |
237 | }, | |
238 | } | |
83c7162d XL |
239 | } |
240 | } | |
241 | ||
242 | impl CastTarget { | |
243 | pub fn pair(a: Reg, b: Reg) -> CastTarget { | |
244 | CastTarget { | |
a2a8927a | 245 | prefix: [Some(a), None, None, None, None, None, None, None], |
dfeec247 | 246 | rest: Uniform::from(b), |
a2a8927a XL |
247 | attrs: ArgAttributes { |
248 | regular: ArgAttribute::default(), | |
249 | arg_ext: ArgExtension::None, | |
250 | pointee_size: Size::ZERO, | |
251 | pointee_align: None, | |
252 | }, | |
83c7162d XL |
253 | } |
254 | } | |
255 | ||
a2a8927a XL |
256 | pub fn size<C: HasDataLayout>(&self, _cx: &C) -> Size { |
257 | let mut size = self.rest.total; | |
258 | for i in 0..self.prefix.iter().count() { | |
259 | match self.prefix[i] { | |
487cf647 | 260 | Some(v) => size += v.size, |
a2a8927a XL |
261 | None => {} |
262 | } | |
263 | } | |
264 | return size; | |
83c7162d XL |
265 | } |
266 | ||
a1dfa0c6 | 267 | pub fn align<C: HasDataLayout>(&self, cx: &C) -> Align { |
dfeec247 XL |
268 | self.prefix |
269 | .iter() | |
a2a8927a | 270 | .filter_map(|x| x.map(|reg| reg.align(cx))) |
dfeec247 XL |
271 | .fold(cx.data_layout().aggregate_align.abi.max(self.rest.align(cx)), |acc, align| { |
272 | acc.max(align) | |
273 | }) | |
83c7162d XL |
274 | } |
275 | } | |
276 | ||
74b04a01 | 277 | /// Return value from the `homogeneous_aggregate` test function. |
9fa01778 XL |
278 | #[derive(Copy, Clone, Debug)] |
279 | pub enum HomogeneousAggregate { | |
280 | /// Yes, all the "leaf fields" of this struct are passed in the | |
281 | /// same way (specified in the `Reg` value). | |
282 | Homogeneous(Reg), | |
283 | ||
9fa01778 XL |
284 | /// There are no leaf fields at all. |
285 | NoData, | |
286 | } | |
287 | ||
74b04a01 XL |
288 | /// Error from the `homogeneous_aggregate` test function, indicating |
289 | /// there are distinct leaf fields passed in different ways, | |
290 | /// or this is uninhabited. | |
291 | #[derive(Copy, Clone, Debug)] | |
292 | pub struct Heterogeneous; | |
293 | ||
9fa01778 XL |
294 | impl HomogeneousAggregate { |
295 | /// If this is a homogeneous aggregate, returns the homogeneous | |
296 | /// unit, else `None`. | |
297 | pub fn unit(self) -> Option<Reg> { | |
74b04a01 XL |
298 | match self { |
299 | HomogeneousAggregate::Homogeneous(reg) => Some(reg), | |
300 | HomogeneousAggregate::NoData => None, | |
301 | } | |
302 | } | |
303 | ||
304 | /// Try to combine two `HomogeneousAggregate`s, e.g. from two fields in | |
305 | /// the same `struct`. Only succeeds if only one of them has any data, | |
306 | /// or both units are identical. | |
307 | fn merge(self, other: HomogeneousAggregate) -> Result<HomogeneousAggregate, Heterogeneous> { | |
308 | match (self, other) { | |
309 | (x, HomogeneousAggregate::NoData) | (HomogeneousAggregate::NoData, x) => Ok(x), | |
310 | ||
311 | (HomogeneousAggregate::Homogeneous(a), HomogeneousAggregate::Homogeneous(b)) => { | |
312 | if a != b { | |
313 | return Err(Heterogeneous); | |
314 | } | |
315 | Ok(self) | |
316 | } | |
317 | } | |
9fa01778 XL |
318 | } |
319 | } | |
320 | ||
ba9703b0 | 321 | impl<'a, Ty> TyAndLayout<'a, Ty> { |
83c7162d XL |
322 | fn is_aggregate(&self) -> bool { |
323 | match self.abi { | |
dfeec247 XL |
324 | Abi::Uninhabited | Abi::Scalar(_) | Abi::Vector { .. } => false, |
325 | Abi::ScalarPair(..) | Abi::Aggregate { .. } => true, | |
83c7162d XL |
326 | } |
327 | } | |
328 | ||
74b04a01 XL |
329 | /// Returns `Homogeneous` if this layout is an aggregate containing fields of |
330 | /// only a single type (e.g., `(u32, u32)`). Such aggregates are often | |
9fa01778 XL |
331 | /// special-cased in ABIs. |
332 | /// | |
333 | /// Note: We generally ignore fields of zero-sized type when computing | |
334 | /// this value (see #56877). | |
335 | /// | |
336 | /// This is public so that it can be used in unit tests, but | |
337 | /// should generally only be relevant to the ABI details of | |
338 | /// specific targets. | |
74b04a01 | 339 | pub fn homogeneous_aggregate<C>(&self, cx: &C) -> Result<HomogeneousAggregate, Heterogeneous> |
dfeec247 | 340 | where |
94222f64 | 341 | Ty: TyAbiInterface<'a, C> + Copy, |
83c7162d XL |
342 | { |
343 | match self.abi { | |
74b04a01 | 344 | Abi::Uninhabited => Err(Heterogeneous), |
83c7162d XL |
345 | |
346 | // The primitive for this algorithm. | |
c295e0f8 | 347 | Abi::Scalar(scalar) => { |
04454e1e | 348 | let kind = match scalar.primitive() { |
dfeec247 | 349 | abi::Int(..) | abi::Pointer => RegKind::Integer, |
60c5eb7d | 350 | abi::F32 | abi::F64 => RegKind::Float, |
83c7162d | 351 | }; |
74b04a01 | 352 | Ok(HomogeneousAggregate::Homogeneous(Reg { kind, size: self.size })) |
83c7162d XL |
353 | } |
354 | ||
355 | Abi::Vector { .. } => { | |
9fa01778 | 356 | assert!(!self.is_zst()); |
74b04a01 XL |
357 | Ok(HomogeneousAggregate::Homogeneous(Reg { |
358 | kind: RegKind::Vector, | |
359 | size: self.size, | |
360 | })) | |
83c7162d XL |
361 | } |
362 | ||
dfeec247 | 363 | Abi::ScalarPair(..) | Abi::Aggregate { .. } => { |
ba9703b0 | 364 | // Helper for computing `homogeneous_aggregate`, allowing a custom |
74b04a01 XL |
365 | // starting offset (used below for handling variants). |
366 | let from_fields_at = | |
367 | |layout: Self, | |
368 | start: Size| | |
369 | -> Result<(HomogeneousAggregate, Size), Heterogeneous> { | |
370 | let is_union = match layout.fields { | |
ba9703b0 XL |
371 | FieldsShape::Primitive => { |
372 | unreachable!("aggregates can't have `FieldsShape::Primitive`") | |
373 | } | |
374 | FieldsShape::Array { count, .. } => { | |
74b04a01 XL |
375 | assert_eq!(start, Size::ZERO); |
376 | ||
377 | let result = if count > 0 { | |
378 | layout.field(cx, 0).homogeneous_aggregate(cx)? | |
379 | } else { | |
380 | HomogeneousAggregate::NoData | |
381 | }; | |
382 | return Ok((result, layout.size)); | |
383 | } | |
ba9703b0 XL |
384 | FieldsShape::Union(_) => true, |
385 | FieldsShape::Arbitrary { .. } => false, | |
74b04a01 | 386 | }; |
83c7162d | 387 | |
74b04a01 XL |
388 | let mut result = HomogeneousAggregate::NoData; |
389 | let mut total = start; | |
83c7162d | 390 | |
74b04a01 XL |
391 | for i in 0..layout.fields.count() { |
392 | if !is_union && total != layout.fields.offset(i) { | |
393 | return Err(Heterogeneous); | |
394 | } | |
9fa01778 | 395 | |
74b04a01 XL |
396 | let field = layout.field(cx, i); |
397 | ||
398 | result = result.merge(field.homogeneous_aggregate(cx)?)?; | |
399 | ||
400 | // Keep track of the offset (without padding). | |
401 | let size = field.size; | |
402 | if is_union { | |
403 | total = total.max(size); | |
404 | } else { | |
405 | total += size; | |
83c7162d XL |
406 | } |
407 | } | |
83c7162d | 408 | |
74b04a01 XL |
409 | Ok((result, total)) |
410 | }; | |
411 | ||
412 | let (mut result, mut total) = from_fields_at(*self, Size::ZERO)?; | |
413 | ||
414 | match &self.variants { | |
415 | abi::Variants::Single { .. } => {} | |
416 | abi::Variants::Multiple { variants, .. } => { | |
417 | // Treat enum variants like union members. | |
418 | // HACK(eddyb) pretend the `enum` field (discriminant) | |
419 | // is at the start of every variant (otherwise the gap | |
420 | // at the start of all variants would disqualify them). | |
421 | // | |
422 | // NB: for all tagged `enum`s (which include all non-C-like | |
423 | // `enum`s with defined FFI representation), this will | |
424 | // match the homogeneous computation on the equivalent | |
425 | // `struct { tag; union { variant1; ... } }` and/or | |
426 | // `union { struct { tag; variant1; } ... }` | |
427 | // (the offsets of variant fields should be identical | |
428 | // between the two for either to be a homogeneous aggregate). | |
429 | let variant_start = total; | |
430 | for variant_idx in variants.indices() { | |
431 | let (variant_result, variant_total) = | |
432 | from_fields_at(self.for_variant(cx, variant_idx), variant_start)?; | |
433 | ||
434 | result = result.merge(variant_result)?; | |
435 | total = total.max(variant_total); | |
436 | } | |
83c7162d XL |
437 | } |
438 | } | |
439 | ||
440 | // There needs to be no padding. | |
441 | if total != self.size { | |
74b04a01 | 442 | Err(Heterogeneous) |
83c7162d | 443 | } else { |
9fa01778 | 444 | match result { |
74b04a01 | 445 | HomogeneousAggregate::Homogeneous(_) => { |
9fa01778 | 446 | assert_ne!(total, Size::ZERO); |
9fa01778 | 447 | } |
74b04a01 | 448 | HomogeneousAggregate::NoData => { |
9fa01778 | 449 | assert_eq!(total, Size::ZERO); |
9fa01778 XL |
450 | } |
451 | } | |
74b04a01 | 452 | Ok(result) |
83c7162d XL |
453 | } |
454 | } | |
455 | } | |
456 | } | |
457 | } | |
458 | ||
459 | /// Information about how to pass an argument to, | |
460 | /// or return a value from, a function, under some ABI. | |
c295e0f8 | 461 | #[derive(PartialEq, Eq, Hash, Debug, HashStable_Generic)] |
60c5eb7d | 462 | pub struct ArgAbi<'a, Ty> { |
ba9703b0 | 463 | pub layout: TyAndLayout<'a, Ty>, |
83c7162d XL |
464 | pub mode: PassMode, |
465 | } | |
466 | ||
60c5eb7d | 467 | impl<'a, Ty> ArgAbi<'a, Ty> { |
5869c6ff XL |
468 | pub fn new( |
469 | cx: &impl HasDataLayout, | |
470 | layout: TyAndLayout<'a, Ty>, | |
c295e0f8 | 471 | scalar_attrs: impl Fn(&TyAndLayout<'a, Ty>, abi::Scalar, Size) -> ArgAttributes, |
5869c6ff | 472 | ) -> Self { |
c295e0f8 | 473 | let mode = match layout.abi { |
5869c6ff XL |
474 | Abi::Uninhabited => PassMode::Ignore, |
475 | Abi::Scalar(scalar) => PassMode::Direct(scalar_attrs(&layout, scalar, Size::ZERO)), | |
476 | Abi::ScalarPair(a, b) => PassMode::Pair( | |
477 | scalar_attrs(&layout, a, Size::ZERO), | |
04454e1e | 478 | scalar_attrs(&layout, b, a.size(cx).align_to(b.align(cx).abi)), |
5869c6ff XL |
479 | ), |
480 | Abi::Vector { .. } => PassMode::Direct(ArgAttributes::new()), | |
481 | Abi::Aggregate { .. } => PassMode::Direct(ArgAttributes::new()), | |
482 | }; | |
f2b60f7d | 483 | ArgAbi { layout, mode } |
83c7162d XL |
484 | } |
485 | ||
5869c6ff | 486 | fn indirect_pass_mode(layout: &TyAndLayout<'a, Ty>) -> PassMode { |
83c7162d XL |
487 | let mut attrs = ArgAttributes::new(); |
488 | ||
489 | // For non-immediate arguments the callee gets its own copy of | |
490 | // the value on the stack, so there are no aliases. It's also | |
491 | // program-invisible so can't possibly capture | |
5099ac24 FG |
492 | attrs |
493 | .set(ArgAttribute::NoAlias) | |
494 | .set(ArgAttribute::NoCapture) | |
495 | .set(ArgAttribute::NonNull) | |
496 | .set(ArgAttribute::NoUndef); | |
5869c6ff | 497 | attrs.pointee_size = layout.size; |
83c7162d XL |
498 | // FIXME(eddyb) We should be doing this, but at least on |
499 | // i686-pc-windows-msvc, it results in wrong stack offsets. | |
5869c6ff | 500 | // attrs.pointee_align = Some(layout.align.abi); |
83c7162d | 501 | |
5869c6ff | 502 | let extra_attrs = layout.is_unsized().then_some(ArgAttributes::new()); |
b7449926 | 503 | |
5869c6ff XL |
504 | PassMode::Indirect { attrs, extra_attrs, on_stack: false } |
505 | } | |
506 | ||
507 | pub fn make_indirect(&mut self) { | |
508 | match self.mode { | |
509 | PassMode::Direct(_) | PassMode::Pair(_, _) => {} | |
510 | PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: false } => return, | |
511 | _ => panic!("Tried to make {:?} indirect", self.mode), | |
512 | } | |
513 | ||
514 | self.mode = Self::indirect_pass_mode(&self.layout); | |
83c7162d XL |
515 | } |
516 | ||
517 | pub fn make_indirect_byval(&mut self) { | |
518 | self.make_indirect(); | |
519 | match self.mode { | |
fc512014 XL |
520 | PassMode::Indirect { attrs: _, extra_attrs: _, ref mut on_stack } => { |
521 | *on_stack = true; | |
83c7162d | 522 | } |
dfeec247 | 523 | _ => unreachable!(), |
83c7162d XL |
524 | } |
525 | } | |
526 | ||
527 | pub fn extend_integer_width_to(&mut self, bits: u64) { | |
528 | // Only integers have signedness | |
c295e0f8 | 529 | if let Abi::Scalar(scalar) = self.layout.abi { |
04454e1e | 530 | if let abi::Int(i, signed) = scalar.primitive() { |
83c7162d XL |
531 | if i.size().bits() < bits { |
532 | if let PassMode::Direct(ref mut attrs) = self.mode { | |
fc512014 XL |
533 | if signed { |
534 | attrs.ext(ArgExtension::Sext) | |
535 | } else { | |
536 | attrs.ext(ArgExtension::Zext) | |
537 | }; | |
83c7162d XL |
538 | } |
539 | } | |
540 | } | |
541 | } | |
542 | } | |
543 | ||
544 | pub fn cast_to<T: Into<CastTarget>>(&mut self, target: T) { | |
f2b60f7d | 545 | self.mode = PassMode::Cast(Box::new(target.into()), false); |
83c7162d XL |
546 | } |
547 | ||
f2b60f7d FG |
548 | pub fn cast_to_and_pad_i32<T: Into<CastTarget>>(&mut self, target: T, pad_i32: bool) { |
549 | self.mode = PassMode::Cast(Box::new(target.into()), pad_i32); | |
83c7162d XL |
550 | } |
551 | ||
552 | pub fn is_indirect(&self) -> bool { | |
5869c6ff | 553 | matches!(self.mode, PassMode::Indirect { .. }) |
b7449926 XL |
554 | } |
555 | ||
556 | pub fn is_sized_indirect(&self) -> bool { | |
fc512014 | 557 | matches!(self.mode, PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ }) |
b7449926 XL |
558 | } |
559 | ||
560 | pub fn is_unsized_indirect(&self) -> bool { | |
fc512014 | 561 | matches!(self.mode, PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ }) |
83c7162d XL |
562 | } |
563 | ||
564 | pub fn is_ignore(&self) -> bool { | |
29967ef6 | 565 | matches!(self.mode, PassMode::Ignore) |
83c7162d XL |
566 | } |
567 | } | |
568 | ||
c295e0f8 | 569 | #[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)] |
83c7162d | 570 | pub enum Conv { |
60c5eb7d XL |
571 | // General language calling conventions, for which every target |
572 | // should have its own backend (e.g. LLVM) support. | |
83c7162d | 573 | C, |
60c5eb7d XL |
574 | Rust, |
575 | ||
923072b8 FG |
576 | /// For things unlikely to be called, where smaller caller codegen is |
577 | /// preferred over raw speed. | |
578 | /// Stronger than just `#[cold]` because `fn` pointers might be incompatible. | |
579 | RustCold, | |
580 | ||
60c5eb7d | 581 | // Target-specific calling conventions. |
83c7162d | 582 | ArmAapcs, |
5869c6ff | 583 | CCmseNonSecureCall, |
83c7162d XL |
584 | |
585 | Msp430Intr, | |
586 | ||
587 | PtxKernel, | |
588 | ||
589 | X86Fastcall, | |
590 | X86Intr, | |
591 | X86Stdcall, | |
592 | X86ThisCall, | |
593 | X86VectorCall, | |
594 | ||
595 | X86_64SysV, | |
596 | X86_64Win64, | |
8faf50e0 XL |
597 | |
598 | AmdGpuKernel, | |
f035d41b XL |
599 | AvrInterrupt, |
600 | AvrNonBlockingInterrupt, | |
83c7162d XL |
601 | } |
602 | ||
603 | /// Metadata describing how the arguments to a native function | |
604 | /// should be passed in order to respect the native ABI. | |
605 | /// | |
606 | /// I will do my best to describe this structure, but these | |
607 | /// comments are reverse-engineered and may be inaccurate. -NDM | |
c295e0f8 | 608 | #[derive(PartialEq, Eq, Hash, Debug, HashStable_Generic)] |
60c5eb7d | 609 | pub struct FnAbi<'a, Ty> { |
83c7162d | 610 | /// The LLVM types of each argument. |
f2b60f7d | 611 | pub args: Box<[ArgAbi<'a, Ty>]>, |
83c7162d XL |
612 | |
613 | /// LLVM return type. | |
60c5eb7d | 614 | pub ret: ArgAbi<'a, Ty>, |
83c7162d | 615 | |
532ac7d7 | 616 | pub c_variadic: bool, |
83c7162d | 617 | |
74b04a01 XL |
618 | /// The count of non-variadic arguments. |
619 | /// | |
620 | /// Should only be different from args.len() when c_variadic is true. | |
621 | /// This can be used to know whether an argument is variadic or not. | |
f2b60f7d | 622 | pub fixed_count: u32, |
74b04a01 | 623 | |
83c7162d | 624 | pub conv: Conv, |
ba9703b0 XL |
625 | |
626 | pub can_unwind: bool, | |
83c7162d XL |
627 | } |
628 | ||
c295e0f8 | 629 | /// Error produced by attempting to adjust a `FnAbi`, for a "foreign" ABI. |
5099ac24 | 630 | #[derive(Copy, Clone, Debug, HashStable_Generic)] |
c295e0f8 XL |
631 | pub enum AdjustForForeignAbiError { |
632 | /// Target architecture doesn't support "foreign" (i.e. non-Rust) ABIs. | |
5099ac24 | 633 | Unsupported { arch: Symbol, abi: spec::abi::Abi }, |
c295e0f8 XL |
634 | } |
635 | ||
636 | impl fmt::Display for AdjustForForeignAbiError { | |
637 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { | |
638 | match self { | |
639 | Self::Unsupported { arch, abi } => { | |
9c376795 | 640 | write!(f, "target architecture {arch:?} does not support `extern {abi}` ABI") |
c295e0f8 XL |
641 | } |
642 | } | |
643 | } | |
644 | } | |
645 | ||
60c5eb7d | 646 | impl<'a, Ty> FnAbi<'a, Ty> { |
c295e0f8 XL |
647 | pub fn adjust_for_foreign_abi<C>( |
648 | &mut self, | |
649 | cx: &C, | |
650 | abi: spec::abi::Abi, | |
651 | ) -> Result<(), AdjustForForeignAbiError> | |
dfeec247 | 652 | where |
94222f64 XL |
653 | Ty: TyAbiInterface<'a, C> + Copy, |
654 | C: HasDataLayout + HasTargetSpec, | |
83c7162d | 655 | { |
6a06907d XL |
656 | if abi == spec::abi::Abi::X86Interrupt { |
657 | if let Some(arg) = self.args.first_mut() { | |
658 | arg.make_indirect_byval(); | |
659 | } | |
660 | return Ok(()); | |
661 | } | |
662 | ||
83c7162d XL |
663 | match &cx.target_spec().arch[..] { |
664 | "x86" => { | |
064997fb FG |
665 | let flavor = if let spec::abi::Abi::Fastcall { .. } |
666 | | spec::abi::Abi::Vectorcall { .. } = abi | |
667 | { | |
668 | x86::Flavor::FastcallOrVectorcall | |
83c7162d XL |
669 | } else { |
670 | x86::Flavor::General | |
671 | }; | |
672 | x86::compute_abi_info(cx, self, flavor); | |
dfeec247 | 673 | } |
5099ac24 FG |
674 | "x86_64" => match abi { |
675 | spec::abi::Abi::SysV64 { .. } => x86_64::compute_abi_info(cx, self), | |
676 | spec::abi::Abi::Win64 { .. } => x86_win64::compute_abi_info(self), | |
677 | _ => { | |
678 | if cx.target_spec().is_like_windows { | |
679 | x86_win64::compute_abi_info(self) | |
680 | } else { | |
681 | x86_64::compute_abi_info(cx, self) | |
682 | } | |
dfeec247 | 683 | } |
5099ac24 | 684 | }, |
f2b60f7d FG |
685 | "aarch64" => { |
686 | let param_policy = if cx.target_spec().is_like_osx { | |
687 | aarch64::ParamExtension::ExtendTo32Bits | |
688 | } else { | |
689 | aarch64::ParamExtension::NoExtension | |
690 | }; | |
691 | aarch64::compute_abi_info(cx, self, param_policy) | |
692 | } | |
b7449926 | 693 | "amdgpu" => amdgpu::compute_abi_info(cx, self), |
83c7162d | 694 | "arm" => arm::compute_abi_info(cx, self), |
f035d41b | 695 | "avr" => avr::compute_abi_info(self), |
487cf647 | 696 | "loongarch64" => loongarch::compute_abi_info(cx, self), |
c295e0f8 | 697 | "m68k" => m68k::compute_abi_info(self), |
83c7162d XL |
698 | "mips" => mips::compute_abi_info(cx, self), |
699 | "mips64" => mips64::compute_abi_info(cx, self), | |
60c5eb7d | 700 | "powerpc" => powerpc::compute_abi_info(self), |
83c7162d XL |
701 | "powerpc64" => powerpc64::compute_abi_info(cx, self), |
702 | "s390x" => s390x::compute_abi_info(cx, self), | |
83c7162d XL |
703 | "msp430" => msp430::compute_abi_info(self), |
704 | "sparc" => sparc::compute_abi_info(cx, self), | |
705 | "sparc64" => sparc64::compute_abi_info(cx, self), | |
04454e1e FG |
706 | "nvptx64" => { |
707 | if cx.target_spec().adjust_abi(abi) == spec::abi::Abi::PtxKernel { | |
708 | nvptx64::compute_ptx_kernel_abi_info(cx, self) | |
709 | } else { | |
710 | nvptx64::compute_abi_info(self) | |
711 | } | |
712 | } | |
83c7162d | 713 | "hexagon" => hexagon::compute_abi_info(self), |
74b04a01 | 714 | "riscv32" | "riscv64" => riscv::compute_abi_info(cx, self), |
cdc7bbd5 XL |
715 | "wasm32" | "wasm64" => { |
716 | if cx.target_spec().adjust_abi(abi) == spec::abi::Abi::Wasm { | |
717 | wasm::compute_wasm_abi_info(self) | |
718 | } else { | |
719 | wasm::compute_c_abi_info(cx, self) | |
720 | } | |
721 | } | |
722 | "asmjs" => wasm::compute_c_abi_info(cx, self), | |
17df50a5 | 723 | "bpf" => bpf::compute_abi_info(self), |
c295e0f8 | 724 | arch => { |
5099ac24 FG |
725 | return Err(AdjustForForeignAbiError::Unsupported { |
726 | arch: Symbol::intern(arch), | |
727 | abi, | |
728 | }); | |
c295e0f8 | 729 | } |
83c7162d XL |
730 | } |
731 | ||
83c7162d XL |
732 | Ok(()) |
733 | } | |
734 | } | |
f2b60f7d | 735 | |
487cf647 FG |
736 | impl FromStr for Conv { |
737 | type Err = String; | |
738 | ||
739 | fn from_str(s: &str) -> Result<Self, Self::Err> { | |
740 | match s { | |
741 | "C" => Ok(Conv::C), | |
742 | "Rust" => Ok(Conv::Rust), | |
743 | "RustCold" => Ok(Conv::Rust), | |
744 | "ArmAapcs" => Ok(Conv::ArmAapcs), | |
745 | "CCmseNonSecureCall" => Ok(Conv::CCmseNonSecureCall), | |
746 | "Msp430Intr" => Ok(Conv::Msp430Intr), | |
747 | "PtxKernel" => Ok(Conv::PtxKernel), | |
748 | "X86Fastcall" => Ok(Conv::X86Fastcall), | |
749 | "X86Intr" => Ok(Conv::X86Intr), | |
750 | "X86Stdcall" => Ok(Conv::X86Stdcall), | |
751 | "X86ThisCall" => Ok(Conv::X86ThisCall), | |
752 | "X86VectorCall" => Ok(Conv::X86VectorCall), | |
753 | "X86_64SysV" => Ok(Conv::X86_64SysV), | |
754 | "X86_64Win64" => Ok(Conv::X86_64Win64), | |
755 | "AmdGpuKernel" => Ok(Conv::AmdGpuKernel), | |
756 | "AvrInterrupt" => Ok(Conv::AvrInterrupt), | |
757 | "AvrNonBlockingInterrupt" => Ok(Conv::AvrNonBlockingInterrupt), | |
9c376795 | 758 | _ => Err(format!("'{s}' is not a valid value for entry function call convetion.")), |
487cf647 FG |
759 | } |
760 | } | |
761 | } | |
762 | ||
f2b60f7d FG |
763 | // Some types are used a lot. Make sure they don't unintentionally get bigger. |
764 | #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))] | |
765 | mod size_asserts { | |
766 | use super::*; | |
767 | use rustc_data_structures::static_assert_size; | |
2b03887a | 768 | // tidy-alphabetical-start |
f2b60f7d FG |
769 | static_assert_size!(ArgAbi<'_, usize>, 56); |
770 | static_assert_size!(FnAbi<'_, usize>, 80); | |
2b03887a | 771 | // tidy-alphabetical-end |
f2b60f7d | 772 | } |