]>
Commit | Line | Data |
---|---|---|
29967ef6 XL |
1 | //! Codegen `extern "platform-intrinsic"` intrinsics. |
2 | ||
3 | use super::*; | |
4 | use crate::prelude::*; | |
5 | ||
6 | pub(super) fn codegen_simd_intrinsic_call<'tcx>( | |
6a06907d | 7 | fx: &mut FunctionCx<'_, '_, 'tcx>, |
29967ef6 XL |
8 | instance: Instance<'tcx>, |
9 | args: &[mir::Operand<'tcx>], | |
10 | ret: CPlace<'tcx>, | |
11 | span: Span, | |
12 | ) { | |
13 | let def_id = instance.def_id(); | |
14 | let substs = instance.substs; | |
15 | ||
16 | let intrinsic = fx.tcx.item_name(def_id).as_str(); | |
17 | let intrinsic = &intrinsic[..]; | |
18 | ||
19 | intrinsic_match! { | |
20 | fx, intrinsic, substs, args, | |
21 | _ => { | |
22 | fx.tcx.sess.span_fatal(span, &format!("Unknown SIMD intrinsic {}", intrinsic)); | |
23 | }; | |
24 | ||
25 | simd_cast, (c a) { | |
26 | validate_simd_type!(fx, intrinsic, span, a.layout().ty); | |
27 | simd_for_each_lane(fx, a, ret, |fx, lane_layout, ret_lane_layout, lane| { | |
28 | let ret_lane_ty = fx.clif_type(ret_lane_layout.ty).unwrap(); | |
29 | ||
30 | let from_signed = type_sign(lane_layout.ty); | |
31 | let to_signed = type_sign(ret_lane_layout.ty); | |
32 | ||
33 | let ret_lane = clif_int_or_float_cast(fx, lane, from_signed, ret_lane_ty, to_signed); | |
34 | CValue::by_val(ret_lane, ret_lane_layout) | |
35 | }); | |
36 | }; | |
37 | ||
29967ef6 XL |
38 | simd_eq, (c x, c y) { |
39 | validate_simd_type!(fx, intrinsic, span, x.layout().ty); | |
fc512014 | 40 | simd_cmp!(fx, Equal|Equal(x, y) -> ret); |
29967ef6 XL |
41 | }; |
42 | simd_ne, (c x, c y) { | |
43 | validate_simd_type!(fx, intrinsic, span, x.layout().ty); | |
fc512014 | 44 | simd_cmp!(fx, NotEqual|NotEqual(x, y) -> ret); |
29967ef6 XL |
45 | }; |
46 | simd_lt, (c x, c y) { | |
47 | validate_simd_type!(fx, intrinsic, span, x.layout().ty); | |
fc512014 | 48 | simd_cmp!(fx, UnsignedLessThan|SignedLessThan|LessThan(x, y) -> ret); |
29967ef6 XL |
49 | }; |
50 | simd_le, (c x, c y) { | |
51 | validate_simd_type!(fx, intrinsic, span, x.layout().ty); | |
fc512014 | 52 | simd_cmp!(fx, UnsignedLessThanOrEqual|SignedLessThanOrEqual|LessThanOrEqual(x, y) -> ret); |
29967ef6 XL |
53 | }; |
54 | simd_gt, (c x, c y) { | |
55 | validate_simd_type!(fx, intrinsic, span, x.layout().ty); | |
fc512014 | 56 | simd_cmp!(fx, UnsignedGreaterThan|SignedGreaterThan|GreaterThan(x, y) -> ret); |
29967ef6 XL |
57 | }; |
58 | simd_ge, (c x, c y) { | |
59 | validate_simd_type!(fx, intrinsic, span, x.layout().ty); | |
fc512014 XL |
60 | simd_cmp!( |
61 | fx, | |
62 | UnsignedGreaterThanOrEqual|SignedGreaterThanOrEqual|GreaterThanOrEqual | |
63 | (x, y) -> ret | |
64 | ); | |
29967ef6 XL |
65 | }; |
66 | ||
67 | // simd_shuffle32<T, U>(x: T, y: T, idx: [u32; 32]) -> U | |
68 | _ if intrinsic.starts_with("simd_shuffle"), (c x, c y, o idx) { | |
69 | validate_simd_type!(fx, intrinsic, span, x.layout().ty); | |
70 | ||
71 | let n: u16 = intrinsic["simd_shuffle".len()..].parse().unwrap(); | |
72 | ||
73 | assert_eq!(x.layout(), y.layout()); | |
74 | let layout = x.layout(); | |
75 | ||
5869c6ff XL |
76 | let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx); |
77 | let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); | |
29967ef6 | 78 | |
5869c6ff XL |
79 | assert_eq!(lane_ty, ret_lane_ty); |
80 | assert_eq!(u64::from(n), ret_lane_count); | |
29967ef6 XL |
81 | |
82 | let total_len = lane_count * 2; | |
83 | ||
84 | let indexes = { | |
85 | use rustc_middle::mir::interpret::*; | |
86 | let idx_const = crate::constant::mir_operand_get_const_val(fx, idx).expect("simd_shuffle* idx not const"); | |
87 | ||
6a06907d XL |
88 | let idx_bytes = match idx_const { |
89 | ConstValue::ByRef { alloc, offset } => { | |
29967ef6 XL |
90 | let ptr = Pointer::new(AllocId(0 /* dummy */), offset); |
91 | let size = Size::from_bytes(4 * u64::from(ret_lane_count) /* size_of([u32; ret_lane_count]) */); | |
92 | alloc.get_bytes(fx, ptr, size).unwrap() | |
93 | } | |
94 | _ => unreachable!("{:?}", idx_const), | |
95 | }; | |
96 | ||
97 | (0..ret_lane_count).map(|i| { | |
98 | let i = usize::try_from(i).unwrap(); | |
99 | let idx = rustc_middle::mir::interpret::read_target_uint( | |
100 | fx.tcx.data_layout.endian, | |
101 | &idx_bytes[4*i.. 4*i + 4], | |
102 | ).expect("read_target_uint"); | |
103 | u16::try_from(idx).expect("try_from u32") | |
104 | }).collect::<Vec<u16>>() | |
105 | }; | |
106 | ||
107 | for &idx in &indexes { | |
5869c6ff | 108 | assert!(u64::from(idx) < total_len, "idx {} out of range 0..{}", idx, total_len); |
29967ef6 XL |
109 | } |
110 | ||
111 | for (out_idx, in_idx) in indexes.into_iter().enumerate() { | |
5869c6ff | 112 | let in_lane = if u64::from(in_idx) < lane_count { |
fc512014 | 113 | x.value_field(fx, mir::Field::new(in_idx.into())) |
29967ef6 | 114 | } else { |
5869c6ff | 115 | y.value_field(fx, mir::Field::new(usize::from(in_idx) - usize::try_from(lane_count).unwrap())) |
29967ef6 XL |
116 | }; |
117 | let out_lane = ret.place_field(fx, mir::Field::new(out_idx)); | |
118 | out_lane.write_cvalue(fx, in_lane); | |
119 | } | |
120 | }; | |
121 | ||
122 | simd_insert, (c base, o idx, c val) { | |
123 | // FIXME validate | |
124 | let idx_const = if let Some(idx_const) = crate::constant::mir_operand_get_const_val(fx, idx) { | |
125 | idx_const | |
126 | } else { | |
127 | fx.tcx.sess.span_fatal( | |
128 | span, | |
129 | "Index argument for `simd_insert` is not a constant", | |
130 | ); | |
131 | }; | |
132 | ||
6a06907d | 133 | let idx = idx_const.try_to_bits(Size::from_bytes(4 /* u32*/)).unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const)); |
5869c6ff | 134 | let (lane_count, _lane_ty) = base.layout().ty.simd_size_and_type(fx.tcx); |
29967ef6 XL |
135 | if idx >= lane_count.into() { |
136 | fx.tcx.sess.span_fatal(fx.mir.span, &format!("[simd_insert] idx {} >= lane_count {}", idx, lane_count)); | |
137 | } | |
138 | ||
139 | ret.write_cvalue(fx, base); | |
140 | let ret_lane = ret.place_field(fx, mir::Field::new(idx.try_into().unwrap())); | |
141 | ret_lane.write_cvalue(fx, val); | |
142 | }; | |
143 | ||
144 | simd_extract, (c v, o idx) { | |
145 | validate_simd_type!(fx, intrinsic, span, v.layout().ty); | |
146 | let idx_const = if let Some(idx_const) = crate::constant::mir_operand_get_const_val(fx, idx) { | |
147 | idx_const | |
148 | } else { | |
fc512014 | 149 | fx.tcx.sess.span_warn( |
29967ef6 XL |
150 | span, |
151 | "Index argument for `simd_extract` is not a constant", | |
152 | ); | |
fc512014 XL |
153 | let res = crate::trap::trap_unimplemented_ret_value( |
154 | fx, | |
155 | ret.layout(), | |
156 | "Index argument for `simd_extract` is not a constant", | |
157 | ); | |
158 | ret.write_cvalue(fx, res); | |
159 | return; | |
29967ef6 XL |
160 | }; |
161 | ||
6a06907d | 162 | let idx = idx_const.try_to_bits(Size::from_bytes(4 /* u32*/)).unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const)); |
5869c6ff | 163 | let (lane_count, _lane_ty) = v.layout().ty.simd_size_and_type(fx.tcx); |
29967ef6 XL |
164 | if idx >= lane_count.into() { |
165 | fx.tcx.sess.span_fatal(fx.mir.span, &format!("[simd_extract] idx {} >= lane_count {}", idx, lane_count)); | |
166 | } | |
167 | ||
168 | let ret_lane = v.value_field(fx, mir::Field::new(idx.try_into().unwrap())); | |
169 | ret.write_cvalue(fx, ret_lane); | |
170 | }; | |
171 | ||
172 | simd_add, (c x, c y) { | |
173 | validate_simd_type!(fx, intrinsic, span, x.layout().ty); | |
174 | simd_int_flt_binop!(fx, iadd|fadd(x, y) -> ret); | |
175 | }; | |
176 | simd_sub, (c x, c y) { | |
177 | validate_simd_type!(fx, intrinsic, span, x.layout().ty); | |
178 | simd_int_flt_binop!(fx, isub|fsub(x, y) -> ret); | |
179 | }; | |
180 | simd_mul, (c x, c y) { | |
181 | validate_simd_type!(fx, intrinsic, span, x.layout().ty); | |
182 | simd_int_flt_binop!(fx, imul|fmul(x, y) -> ret); | |
183 | }; | |
184 | simd_div, (c x, c y) { | |
185 | validate_simd_type!(fx, intrinsic, span, x.layout().ty); | |
186 | simd_int_flt_binop!(fx, udiv|sdiv|fdiv(x, y) -> ret); | |
187 | }; | |
188 | simd_shl, (c x, c y) { | |
189 | validate_simd_type!(fx, intrinsic, span, x.layout().ty); | |
190 | simd_int_binop!(fx, ishl(x, y) -> ret); | |
191 | }; | |
192 | simd_shr, (c x, c y) { | |
193 | validate_simd_type!(fx, intrinsic, span, x.layout().ty); | |
194 | simd_int_binop!(fx, ushr|sshr(x, y) -> ret); | |
195 | }; | |
196 | simd_and, (c x, c y) { | |
197 | validate_simd_type!(fx, intrinsic, span, x.layout().ty); | |
198 | simd_int_binop!(fx, band(x, y) -> ret); | |
199 | }; | |
200 | simd_or, (c x, c y) { | |
201 | validate_simd_type!(fx, intrinsic, span, x.layout().ty); | |
202 | simd_int_binop!(fx, bor(x, y) -> ret); | |
203 | }; | |
204 | simd_xor, (c x, c y) { | |
205 | validate_simd_type!(fx, intrinsic, span, x.layout().ty); | |
206 | simd_int_binop!(fx, bxor(x, y) -> ret); | |
207 | }; | |
208 | ||
209 | simd_fma, (c a, c b, c c) { | |
210 | validate_simd_type!(fx, intrinsic, span, a.layout().ty); | |
211 | assert_eq!(a.layout(), b.layout()); | |
212 | assert_eq!(a.layout(), c.layout()); | |
213 | let layout = a.layout(); | |
214 | ||
5869c6ff XL |
215 | let (lane_count, _lane_ty) = layout.ty.simd_size_and_type(fx.tcx); |
216 | let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); | |
29967ef6 | 217 | assert_eq!(lane_count, ret_lane_count); |
5869c6ff | 218 | let ret_lane_layout = fx.layout_of(ret_lane_ty); |
29967ef6 XL |
219 | |
220 | for lane in 0..lane_count { | |
5869c6ff | 221 | let lane = mir::Field::new(lane.try_into().unwrap()); |
29967ef6 XL |
222 | let a_lane = a.value_field(fx, lane).load_scalar(fx); |
223 | let b_lane = b.value_field(fx, lane).load_scalar(fx); | |
224 | let c_lane = c.value_field(fx, lane).load_scalar(fx); | |
225 | ||
226 | let mul_lane = fx.bcx.ins().fmul(a_lane, b_lane); | |
227 | let res_lane = CValue::by_val(fx.bcx.ins().fadd(mul_lane, c_lane), ret_lane_layout); | |
228 | ||
229 | ret.place_field(fx, lane).write_cvalue(fx, res_lane); | |
230 | } | |
231 | }; | |
232 | ||
233 | simd_fmin, (c x, c y) { | |
234 | validate_simd_type!(fx, intrinsic, span, x.layout().ty); | |
235 | simd_flt_binop!(fx, fmin(x, y) -> ret); | |
236 | }; | |
237 | simd_fmax, (c x, c y) { | |
238 | validate_simd_type!(fx, intrinsic, span, x.layout().ty); | |
239 | simd_flt_binop!(fx, fmax(x, y) -> ret); | |
240 | }; | |
241 | ||
fc512014 XL |
242 | simd_reduce_add_ordered | simd_reduce_add_unordered, (c v) { |
243 | validate_simd_type!(fx, intrinsic, span, v.layout().ty); | |
244 | simd_reduce(fx, v, ret, |fx, lane_layout, a, b| { | |
245 | if lane_layout.ty.is_floating_point() { | |
246 | fx.bcx.ins().fadd(a, b) | |
247 | } else { | |
248 | fx.bcx.ins().iadd(a, b) | |
249 | } | |
250 | }); | |
251 | }; | |
252 | ||
253 | simd_reduce_mul_ordered | simd_reduce_mul_unordered, (c v) { | |
254 | validate_simd_type!(fx, intrinsic, span, v.layout().ty); | |
255 | simd_reduce(fx, v, ret, |fx, lane_layout, a, b| { | |
256 | if lane_layout.ty.is_floating_point() { | |
257 | fx.bcx.ins().fmul(a, b) | |
258 | } else { | |
259 | fx.bcx.ins().imul(a, b) | |
260 | } | |
261 | }); | |
262 | }; | |
263 | ||
264 | simd_reduce_all, (c v) { | |
265 | validate_simd_type!(fx, intrinsic, span, v.layout().ty); | |
266 | simd_reduce_bool(fx, v, ret, |fx, a, b| fx.bcx.ins().band(a, b)); | |
267 | }; | |
268 | ||
269 | simd_reduce_any, (c v) { | |
270 | validate_simd_type!(fx, intrinsic, span, v.layout().ty); | |
271 | simd_reduce_bool(fx, v, ret, |fx, a, b| fx.bcx.ins().bor(a, b)); | |
272 | }; | |
273 | ||
29967ef6 XL |
274 | // simd_fabs |
275 | // simd_saturating_add | |
276 | // simd_bitmask | |
277 | // simd_select | |
29967ef6 | 278 | // simd_rem |
6a06907d | 279 | // simd_neg |
29967ef6 XL |
280 | } |
281 | } |