]>
Commit | Line | Data |
---|---|---|
223e47cc LB |
1 | // The classification code for the x86_64 ABI is taken from the clay language |
2 | // https://github.com/jckarter/clay/blob/master/compiler/src/externals.cpp | |
3 | ||
60c5eb7d | 4 | use crate::abi::call::{ArgAbi, CastTarget, FnAbi, Reg, RegKind}; |
9fa01778 | 5 | use crate::abi::{self, Abi, HasDataLayout, LayoutOf, Size, TyLayout, TyLayoutMethods}; |
970d7e83 | 6 | |
2c00a5a8 | 7 | /// Classification of "eightbyte" components. |
0731742a | 8 | // N.B., the order of the variants is from general to specific, |
2c00a5a8 XL |
9 | // such that `unify(a, b)` is the "smaller" of `a` and `b`. |
10 | #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)] | |
cc61c64b | 11 | enum Class { |
cc61c64b XL |
12 | Int, |
13 | Sse, | |
14 | SseUp | |
970d7e83 LB |
15 | } |
16 | ||
cc61c64b XL |
17 | #[derive(Clone, Copy, Debug)] |
18 | struct Memory; | |
970d7e83 | 19 | |
abe05a73 XL |
20 | // Currently supported vector size (AVX-512). |
21 | const LARGEST_VECTOR_SIZE: usize = 512; | |
cc61c64b | 22 | const MAX_EIGHTBYTES: usize = LARGEST_VECTOR_SIZE / 64; |
223e47cc | 23 | |
60c5eb7d | 24 | fn classify_arg<'a, Ty, C>(cx: &C, arg: &ArgAbi<'a, Ty>) |
83c7162d XL |
25 | -> Result<[Option<Class>; MAX_EIGHTBYTES], Memory> |
26 | where Ty: TyLayoutMethods<'a, C> + Copy, | |
27 | C: LayoutOf<Ty = Ty, TyLayout = TyLayout<'a, Ty>> + HasDataLayout | |
28 | { | |
a1dfa0c6 | 29 | fn classify<'a, Ty, C>(cx: &C, layout: TyLayout<'a, Ty>, |
83c7162d XL |
30 | cls: &mut [Option<Class>], off: Size) -> Result<(), Memory> |
31 | where Ty: TyLayoutMethods<'a, C> + Copy, | |
32 | C: LayoutOf<Ty = Ty, TyLayout = TyLayout<'a, Ty>> + HasDataLayout | |
33 | { | |
a1dfa0c6 | 34 | if !off.is_aligned(layout.align.abi) { |
ff7c6d11 | 35 | if !layout.is_zst() { |
cc61c64b | 36 | return Err(Memory); |
1a4d82fc | 37 | } |
cc61c64b | 38 | return Ok(()); |
223e47cc | 39 | } |
223e47cc | 40 | |
2c00a5a8 | 41 | let mut c = match layout.abi { |
83c7162d | 42 | Abi::Uninhabited => return Ok(()), |
ff7c6d11 | 43 | |
83c7162d | 44 | Abi::Scalar(ref scalar) => { |
2c00a5a8 | 45 | match scalar.value { |
83c7162d XL |
46 | abi::Int(..) | |
47 | abi::Pointer => Class::Int, | |
60c5eb7d | 48 | abi::F32 | abi::F64 => Class::Sse |
223e47cc | 49 | } |
970d7e83 | 50 | } |
cc61c64b | 51 | |
83c7162d | 52 | Abi::Vector { .. } => Class::Sse, |
2c00a5a8 | 53 | |
83c7162d XL |
54 | Abi::ScalarPair(..) | |
55 | Abi::Aggregate { .. } => { | |
ff7c6d11 | 56 | match layout.variants { |
83c7162d | 57 | abi::Variants::Single { .. } => { |
ff7c6d11 XL |
58 | for i in 0..layout.fields.count() { |
59 | let field_off = off + layout.fields.offset(i); | |
2c00a5a8 | 60 | classify(cx, layout.field(cx, i), cls, field_off)?; |
ff7c6d11 | 61 | } |
2c00a5a8 | 62 | return Ok(()); |
cc61c64b | 63 | } |
532ac7d7 | 64 | abi::Variants::Multiple { .. } => return Err(Memory), |
85aaf69f SL |
65 | } |
66 | } | |
85aaf69f | 67 | |
2c00a5a8 XL |
68 | }; |
69 | ||
70 | // Fill in `cls` for scalars (Int/Sse) and vectors (Sse). | |
71 | let first = (off.bytes() / 8) as usize; | |
72 | let last = ((off.bytes() + layout.size.bytes() - 1) / 8) as usize; | |
73 | for cls in &mut cls[first..=last] { | |
74 | *cls = Some(cls.map_or(c, |old| old.min(c))); | |
75 | ||
76 | // Everything after the first Sse "eightbyte" | |
77 | // component is the upper half of a register. | |
78 | if c == Class::Sse { | |
79 | c = Class::SseUp; | |
80 | } | |
223e47cc | 81 | } |
cc61c64b XL |
82 | |
83 | Ok(()) | |
223e47cc LB |
84 | } |
85 | ||
ff7c6d11 | 86 | let n = ((arg.layout.size.bytes() + 7) / 8) as usize; |
cc61c64b XL |
87 | if n > MAX_EIGHTBYTES { |
88 | return Err(Memory); | |
89 | } | |
90 | ||
2c00a5a8 | 91 | let mut cls = [None; MAX_EIGHTBYTES]; |
94b46f34 | 92 | classify(cx, arg.layout, &mut cls, Size::ZERO)?; |
cc61c64b | 93 | if n > 2 { |
2c00a5a8 | 94 | if cls[0] != Some(Class::Sse) { |
cc61c64b XL |
95 | return Err(Memory); |
96 | } | |
2c00a5a8 | 97 | if cls[1..n].iter().any(|&c| c != Some(Class::SseUp)) { |
cc61c64b XL |
98 | return Err(Memory); |
99 | } | |
100 | } else { | |
85aaf69f | 101 | let mut i = 0; |
cc61c64b | 102 | while i < n { |
2c00a5a8 XL |
103 | if cls[i] == Some(Class::SseUp) { |
104 | cls[i] = Some(Class::Sse); | |
105 | } else if cls[i] == Some(Class::Sse) { | |
85aaf69f | 106 | i += 1; |
2c00a5a8 | 107 | while i != n && cls[i] == Some(Class::SseUp) { i += 1; } |
970d7e83 | 108 | } else { |
cc61c64b | 109 | i += 1; |
223e47cc LB |
110 | } |
111 | } | |
112 | } | |
113 | ||
cc61c64b | 114 | Ok(cls) |
223e47cc LB |
115 | } |
116 | ||
2c00a5a8 | 117 | fn reg_component(cls: &[Option<Class>], i: &mut usize, size: Size) -> Option<Reg> { |
cc61c64b XL |
118 | if *i >= cls.len() { |
119 | return None; | |
223e47cc LB |
120 | } |
121 | ||
cc61c64b | 122 | match cls[*i] { |
2c00a5a8 XL |
123 | None => None, |
124 | Some(Class::Int) => { | |
cc61c64b | 125 | *i += 1; |
2c00a5a8 XL |
126 | Some(if size.bytes() < 8 { |
127 | Reg { | |
128 | kind: RegKind::Integer, | |
129 | size | |
130 | } | |
131 | } else { | |
132 | Reg::i64() | |
cc61c64b | 133 | }) |
223e47cc | 134 | } |
2c00a5a8 XL |
135 | Some(Class::Sse) => { |
136 | let vec_len = 1 + cls[*i+1..].iter() | |
137 | .take_while(|&&c| c == Some(Class::SseUp)) | |
138 | .count(); | |
cc61c64b XL |
139 | *i += vec_len; |
140 | Some(if vec_len == 1 { | |
ff7c6d11 | 141 | match size.bytes() { |
cc61c64b XL |
142 | 4 => Reg::f32(), |
143 | _ => Reg::f64() | |
54a0048b | 144 | } |
223e47cc | 145 | } else { |
cc61c64b XL |
146 | Reg { |
147 | kind: RegKind::Vector, | |
ff7c6d11 | 148 | size: Size::from_bytes(8) * (vec_len as u64) |
cc61c64b XL |
149 | } |
150 | }) | |
1a4d82fc | 151 | } |
83c7162d | 152 | Some(c) => unreachable!("reg_component: unhandled class {:?}", c) |
223e47cc | 153 | } |
cc61c64b XL |
154 | } |
155 | ||
2c00a5a8 | 156 | fn cast_target(cls: &[Option<Class>], size: Size) -> CastTarget { |
cc61c64b XL |
157 | let mut i = 0; |
158 | let lo = reg_component(cls, &mut i, size).unwrap(); | |
ff7c6d11 | 159 | let offset = Size::from_bytes(8) * (i as u64); |
2c00a5a8 XL |
160 | let mut target = CastTarget::from(lo); |
161 | if size > offset { | |
162 | if let Some(hi) = reg_component(cls, &mut i, size - offset) { | |
0531ce1d | 163 | target = CastTarget::pair(lo, hi); |
2c00a5a8 XL |
164 | } |
165 | } | |
94b46f34 | 166 | assert_eq!(reg_component(cls, &mut i, Size::ZERO), None); |
cc61c64b XL |
167 | target |
168 | } | |
223e47cc | 169 | |
dc9dc135 XL |
170 | const MAX_INT_REGS: usize = 6; // RDI, RSI, RDX, RCX, R8, R9 |
171 | const MAX_SSE_REGS: usize = 8; // XMM0-7 | |
172 | ||
60c5eb7d | 173 | pub fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>) |
83c7162d XL |
174 | where Ty: TyLayoutMethods<'a, C> + Copy, |
175 | C: LayoutOf<Ty = Ty, TyLayout = TyLayout<'a, Ty>> + HasDataLayout | |
176 | { | |
dc9dc135 XL |
177 | let mut int_regs = MAX_INT_REGS; |
178 | let mut sse_regs = MAX_SSE_REGS; | |
1a4d82fc | 179 | |
60c5eb7d | 180 | let mut x86_64_arg_or_ret = |arg: &mut ArgAbi<'a, Ty>, is_arg: bool| { |
2c00a5a8 | 181 | let mut cls_or_mem = classify_arg(cx, arg); |
cc61c64b | 182 | |
2c00a5a8 XL |
183 | if is_arg { |
184 | if let Ok(cls) = cls_or_mem { | |
dc9dc135 XL |
185 | let mut needed_int = 0; |
186 | let mut needed_sse = 0; | |
2c00a5a8 | 187 | for &c in &cls { |
cc61c64b | 188 | match c { |
2c00a5a8 XL |
189 | Some(Class::Int) => needed_int += 1, |
190 | Some(Class::Sse) => needed_sse += 1, | |
cc61c64b XL |
191 | _ => {} |
192 | } | |
193 | } | |
dc9dc135 XL |
194 | match (int_regs.checked_sub(needed_int), sse_regs.checked_sub(needed_sse)) { |
195 | (Some(left_int), Some(left_sse)) => { | |
196 | int_regs = left_int; | |
197 | sse_regs = left_sse; | |
198 | } | |
199 | _ => { | |
200 | // Not enough registers for this argument, so it will be | |
201 | // passed on the stack, but we only mark aggregates | |
202 | // explicitly as indirect `byval` arguments, as LLVM will | |
203 | // automatically put immediates on the stack itself. | |
204 | if arg.layout.is_aggregate() { | |
205 | cls_or_mem = Err(Memory); | |
206 | } | |
207 | } | |
2c00a5a8 | 208 | } |
cc61c64b | 209 | } |
2c00a5a8 | 210 | } |
cc61c64b | 211 | |
2c00a5a8 XL |
212 | match cls_or_mem { |
213 | Err(Memory) => { | |
214 | if is_arg { | |
215 | arg.make_indirect_byval(); | |
216 | } else { | |
217 | // `sret` parameter thus one less integer register available | |
218 | arg.make_indirect(); | |
dc9dc135 XL |
219 | // NOTE(eddyb) return is handled first, so no registers |
220 | // should've been used yet. | |
221 | assert_eq!(int_regs, MAX_INT_REGS); | |
2c00a5a8 XL |
222 | int_regs -= 1; |
223 | } | |
cc61c64b | 224 | } |
2c00a5a8 XL |
225 | Ok(ref cls) => { |
226 | // split into sized chunks passed individually | |
2c00a5a8 XL |
227 | if arg.layout.is_aggregate() { |
228 | let size = arg.layout.size; | |
229 | arg.cast_to(cast_target(cls, size)) | |
230 | } else { | |
231 | arg.extend_integer_width_to(32); | |
232 | } | |
b039eaaf | 233 | } |
cc61c64b XL |
234 | } |
235 | }; | |
236 | ||
60c5eb7d XL |
237 | if !fn_abi.ret.is_ignore() { |
238 | x86_64_arg_or_ret(&mut fn_abi.ret, false); | |
54a0048b | 239 | } |
1a4d82fc | 240 | |
60c5eb7d | 241 | for arg in &mut fn_abi.args { |
54a0048b | 242 | if arg.is_ignore() { continue; } |
60c5eb7d | 243 | x86_64_arg_or_ret(arg, true); |
b039eaaf | 244 | } |
223e47cc | 245 | } |