]>
Commit | Line | Data |
---|---|---|
923072b8 FG |
1 | use std::cmp::Ordering; |
2 | ||
3 | use gccjit::{BinaryOp, RValue, Type, ToRValue}; | |
c295e0f8 XL |
4 | use rustc_codegen_ssa::base::compare_simd_types; |
5 | use rustc_codegen_ssa::common::{TypeKind, span_invalid_monomorphization_error}; | |
6 | use rustc_codegen_ssa::mir::operand::OperandRef; | |
923072b8 | 7 | use rustc_codegen_ssa::mir::place::PlaceRef; |
c295e0f8 XL |
8 | use rustc_codegen_ssa::traits::{BaseTypeMethods, BuilderMethods}; |
9 | use rustc_hir as hir; | |
10 | use rustc_middle::span_bug; | |
11 | use rustc_middle::ty::layout::HasTyCtxt; | |
12 | use rustc_middle::ty::{self, Ty}; | |
13 | use rustc_span::{Span, Symbol, sym}; | |
923072b8 | 14 | use rustc_target::abi::Align; |
c295e0f8 XL |
15 | |
16 | use crate::builder::Builder; | |
923072b8 | 17 | use crate::intrinsic; |
c295e0f8 XL |
18 | |
19 | pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(bx: &mut Builder<'a, 'gcc, 'tcx>, name: Symbol, callee_ty: Ty<'tcx>, args: &[OperandRef<'tcx, RValue<'gcc>>], ret_ty: Ty<'tcx>, llret_ty: Type<'gcc>, span: Span) -> Result<RValue<'gcc>, ()> { | |
20 | // macros for error handling: | |
923072b8 | 21 | #[allow(unused_macro_rules)] |
c295e0f8 XL |
22 | macro_rules! emit_error { |
23 | ($msg: tt) => { | |
24 | emit_error!($msg, ) | |
25 | }; | |
26 | ($msg: tt, $($fmt: tt)*) => { | |
27 | span_invalid_monomorphization_error( | |
28 | bx.sess(), span, | |
29 | &format!(concat!("invalid monomorphization of `{}` intrinsic: ", $msg), | |
30 | name, $($fmt)*)); | |
31 | } | |
32 | } | |
33 | ||
34 | macro_rules! return_error { | |
35 | ($($fmt: tt)*) => { | |
36 | { | |
37 | emit_error!($($fmt)*); | |
38 | return Err(()); | |
39 | } | |
40 | } | |
41 | } | |
42 | ||
43 | macro_rules! require { | |
44 | ($cond: expr, $($fmt: tt)*) => { | |
45 | if !$cond { | |
46 | return_error!($($fmt)*); | |
47 | } | |
48 | }; | |
49 | } | |
50 | ||
51 | macro_rules! require_simd { | |
52 | ($ty: expr, $position: expr) => { | |
53 | require!($ty.is_simd(), "expected SIMD {} type, found non-SIMD `{}`", $position, $ty) | |
54 | }; | |
55 | } | |
56 | ||
57 | let tcx = bx.tcx(); | |
58 | let sig = | |
59 | tcx.normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), callee_ty.fn_sig(tcx)); | |
60 | let arg_tys = sig.inputs(); | |
923072b8 FG |
61 | |
62 | if name == sym::simd_select_bitmask { | |
63 | require_simd!(arg_tys[1], "argument"); | |
64 | let (len, _) = arg_tys[1].simd_size_and_type(bx.tcx()); | |
65 | ||
66 | let expected_int_bits = (len.max(8) - 1).next_power_of_two(); | |
67 | let expected_bytes = len / 8 + ((len % 8 > 0) as u64); | |
68 | ||
69 | let mask_ty = arg_tys[0]; | |
70 | let mut mask = match mask_ty.kind() { | |
71 | ty::Int(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(), | |
72 | ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(), | |
73 | ty::Array(elem, len) | |
74 | if matches!(elem.kind(), ty::Uint(ty::UintTy::U8)) | |
75 | && len.try_eval_usize(bx.tcx, ty::ParamEnv::reveal_all()) | |
76 | == Some(expected_bytes) => | |
77 | { | |
78 | let place = PlaceRef::alloca(bx, args[0].layout); | |
79 | args[0].val.store(bx, place); | |
80 | let int_ty = bx.type_ix(expected_bytes * 8); | |
81 | let ptr = bx.pointercast(place.llval, bx.cx.type_ptr_to(int_ty)); | |
82 | bx.load(int_ty, ptr, Align::ONE) | |
83 | } | |
84 | _ => return_error!( | |
85 | "invalid bitmask `{}`, expected `u{}` or `[u8; {}]`", | |
86 | mask_ty, | |
87 | expected_int_bits, | |
88 | expected_bytes | |
89 | ), | |
90 | }; | |
91 | ||
92 | let arg1 = args[1].immediate(); | |
93 | let arg1_type = arg1.get_type(); | |
94 | let arg1_vector_type = arg1_type.unqualified().dyncast_vector().expect("vector type"); | |
95 | let arg1_element_type = arg1_vector_type.get_element_type(); | |
96 | ||
97 | let mut elements = vec![]; | |
98 | let one = bx.context.new_rvalue_one(mask.get_type()); | |
99 | for _ in 0..len { | |
100 | let element = bx.context.new_cast(None, mask & one, arg1_element_type); | |
101 | elements.push(element); | |
102 | mask = mask >> one; | |
103 | } | |
104 | let vector_mask = bx.context.new_rvalue_from_vector(None, arg1_type, &elements); | |
105 | ||
106 | return Ok(bx.vector_select(vector_mask, arg1, args[2].immediate())); | |
107 | } | |
c295e0f8 XL |
108 | |
109 | // every intrinsic below takes a SIMD vector as its first argument | |
110 | require_simd!(arg_tys[0], "input"); | |
111 | let in_ty = arg_tys[0]; | |
112 | ||
113 | let comparison = match name { | |
114 | sym::simd_eq => Some(hir::BinOpKind::Eq), | |
115 | sym::simd_ne => Some(hir::BinOpKind::Ne), | |
116 | sym::simd_lt => Some(hir::BinOpKind::Lt), | |
117 | sym::simd_le => Some(hir::BinOpKind::Le), | |
118 | sym::simd_gt => Some(hir::BinOpKind::Gt), | |
119 | sym::simd_ge => Some(hir::BinOpKind::Ge), | |
120 | _ => None, | |
121 | }; | |
122 | ||
123 | let (in_len, in_elem) = arg_tys[0].simd_size_and_type(bx.tcx()); | |
124 | if let Some(cmp_op) = comparison { | |
125 | require_simd!(ret_ty, "return"); | |
126 | ||
127 | let (out_len, out_ty) = ret_ty.simd_size_and_type(bx.tcx()); | |
128 | require!( | |
129 | in_len == out_len, | |
130 | "expected return type with length {} (same as input type `{}`), \ | |
131 | found `{}` with length {}", | |
132 | in_len, | |
133 | in_ty, | |
134 | ret_ty, | |
135 | out_len | |
136 | ); | |
137 | require!( | |
138 | bx.type_kind(bx.element_type(llret_ty)) == TypeKind::Integer, | |
139 | "expected return type with integer elements, found `{}` with non-integer `{}`", | |
140 | ret_ty, | |
141 | out_ty | |
142 | ); | |
143 | ||
144 | return Ok(compare_simd_types( | |
145 | bx, | |
146 | args[0].immediate(), | |
147 | args[1].immediate(), | |
148 | in_elem, | |
149 | llret_ty, | |
150 | cmp_op, | |
151 | )); | |
152 | } | |
153 | ||
923072b8 FG |
154 | if let Some(stripped) = name.as_str().strip_prefix("simd_shuffle") { |
155 | let n: u64 = | |
156 | if stripped.is_empty() { | |
157 | // Make sure this is actually an array, since typeck only checks the length-suffixed | |
158 | // version of this intrinsic. | |
159 | match args[2].layout.ty.kind() { | |
160 | ty::Array(ty, len) if matches!(ty.kind(), ty::Uint(ty::UintTy::U32)) => { | |
161 | len.try_eval_usize(bx.cx.tcx, ty::ParamEnv::reveal_all()).unwrap_or_else(|| { | |
162 | span_bug!(span, "could not evaluate shuffle index array length") | |
163 | }) | |
164 | } | |
165 | _ => return_error!( | |
166 | "simd_shuffle index must be an array of `u32`, got `{}`", | |
167 | args[2].layout.ty | |
168 | ), | |
169 | } | |
170 | } | |
171 | else { | |
172 | stripped.parse().unwrap_or_else(|_| { | |
173 | span_bug!(span, "bad `simd_shuffle` instruction only caught in codegen?") | |
174 | }) | |
175 | }; | |
c295e0f8 XL |
176 | |
177 | require_simd!(ret_ty, "return"); | |
178 | ||
179 | let (out_len, out_ty) = ret_ty.simd_size_and_type(bx.tcx()); | |
180 | require!( | |
181 | out_len == n, | |
182 | "expected return type of length {}, found `{}` with length {}", | |
183 | n, | |
184 | ret_ty, | |
185 | out_len | |
186 | ); | |
187 | require!( | |
188 | in_elem == out_ty, | |
189 | "expected return element type `{}` (element of input `{}`), \ | |
190 | found `{}` with element type `{}`", | |
191 | in_elem, | |
192 | in_ty, | |
193 | ret_ty, | |
194 | out_ty | |
195 | ); | |
196 | ||
197 | let vector = args[2].immediate(); | |
198 | ||
199 | return Ok(bx.shuffle_vector( | |
200 | args[0].immediate(), | |
201 | args[1].immediate(), | |
202 | vector, | |
203 | )); | |
204 | } | |
205 | ||
923072b8 FG |
206 | #[cfg(feature="master")] |
207 | if name == sym::simd_insert { | |
208 | require!( | |
209 | in_elem == arg_tys[2], | |
210 | "expected inserted type `{}` (element of input `{}`), found `{}`", | |
211 | in_elem, | |
212 | in_ty, | |
213 | arg_tys[2] | |
214 | ); | |
215 | let vector = args[0].immediate(); | |
216 | let index = args[1].immediate(); | |
217 | let value = args[2].immediate(); | |
218 | // TODO(antoyo): use a recursive unqualified() here. | |
219 | let vector_type = vector.get_type().unqualified().dyncast_vector().expect("vector type"); | |
220 | let element_type = vector_type.get_element_type(); | |
221 | // NOTE: we cannot cast to an array and assign to its element here because the value might | |
222 | // not be an l-value. So, call a builtin to set the element. | |
223 | // TODO(antoyo): perhaps we could create a new vector or maybe there's a GIMPLE instruction for that? | |
224 | // TODO(antoyo): don't use target specific builtins here. | |
225 | let func_name = | |
226 | match in_len { | |
227 | 2 => { | |
228 | if element_type == bx.i64_type { | |
229 | "__builtin_ia32_vec_set_v2di" | |
230 | } | |
231 | else { | |
232 | unimplemented!(); | |
233 | } | |
234 | }, | |
235 | 4 => { | |
236 | if element_type == bx.i32_type { | |
237 | "__builtin_ia32_vec_set_v4si" | |
238 | } | |
239 | else { | |
240 | unimplemented!(); | |
241 | } | |
242 | }, | |
243 | 8 => { | |
244 | if element_type == bx.i16_type { | |
245 | "__builtin_ia32_vec_set_v8hi" | |
246 | } | |
247 | else { | |
248 | unimplemented!(); | |
249 | } | |
250 | }, | |
251 | _ => unimplemented!("Len: {}", in_len), | |
252 | }; | |
253 | let builtin = bx.context.get_target_builtin_function(func_name); | |
254 | let param1_type = builtin.get_param(0).to_rvalue().get_type(); | |
255 | // TODO(antoyo): perhaps use __builtin_convertvector for vector casting. | |
256 | let vector = bx.cx.bitcast_if_needed(vector, param1_type); | |
257 | let result = bx.context.new_call(None, builtin, &[vector, value, bx.context.new_cast(None, index, bx.int_type)]); | |
258 | // TODO(antoyo): perhaps use __builtin_convertvector for vector casting. | |
259 | return Ok(bx.context.new_bitcast(None, result, vector.get_type())); | |
260 | } | |
261 | ||
262 | #[cfg(feature="master")] | |
263 | if name == sym::simd_extract { | |
264 | require!( | |
265 | ret_ty == in_elem, | |
266 | "expected return type `{}` (element of input `{}`), found `{}`", | |
267 | in_elem, | |
268 | in_ty, | |
269 | ret_ty | |
270 | ); | |
271 | let vector = args[0].immediate(); | |
272 | return Ok(bx.context.new_vector_access(None, vector, args[1].immediate()).to_rvalue()); | |
273 | } | |
274 | ||
275 | if name == sym::simd_select { | |
276 | let m_elem_ty = in_elem; | |
277 | let m_len = in_len; | |
278 | require_simd!(arg_tys[1], "argument"); | |
279 | let (v_len, _) = arg_tys[1].simd_size_and_type(bx.tcx()); | |
280 | require!( | |
281 | m_len == v_len, | |
282 | "mismatched lengths: mask length `{}` != other vector length `{}`", | |
283 | m_len, | |
284 | v_len | |
285 | ); | |
286 | match m_elem_ty.kind() { | |
287 | ty::Int(_) => {} | |
288 | _ => return_error!("mask element type is `{}`, expected `i_`", m_elem_ty), | |
289 | } | |
290 | return Ok(bx.vector_select(args[0].immediate(), args[1].immediate(), args[2].immediate())); | |
291 | } | |
292 | ||
293 | if name == sym::simd_cast { | |
294 | require_simd!(ret_ty, "return"); | |
295 | let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx()); | |
296 | require!( | |
297 | in_len == out_len, | |
298 | "expected return type with length {} (same as input type `{}`), \ | |
299 | found `{}` with length {}", | |
300 | in_len, | |
301 | in_ty, | |
302 | ret_ty, | |
303 | out_len | |
304 | ); | |
305 | // casting cares about nominal type, not just structural type | |
306 | if in_elem == out_elem { | |
307 | return Ok(args[0].immediate()); | |
308 | } | |
309 | ||
310 | enum Style { | |
311 | Float, | |
312 | Int(/* is signed? */ bool), | |
313 | Unsupported, | |
314 | } | |
315 | ||
316 | let (in_style, in_width) = match in_elem.kind() { | |
317 | // vectors of pointer-sized integers should've been | |
318 | // disallowed before here, so this unwrap is safe. | |
319 | ty::Int(i) => ( | |
320 | Style::Int(true), | |
321 | i.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(), | |
322 | ), | |
323 | ty::Uint(u) => ( | |
324 | Style::Int(false), | |
325 | u.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(), | |
326 | ), | |
327 | ty::Float(f) => (Style::Float, f.bit_width()), | |
328 | _ => (Style::Unsupported, 0), | |
329 | }; | |
330 | let (out_style, out_width) = match out_elem.kind() { | |
331 | ty::Int(i) => ( | |
332 | Style::Int(true), | |
333 | i.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(), | |
334 | ), | |
335 | ty::Uint(u) => ( | |
336 | Style::Int(false), | |
337 | u.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(), | |
338 | ), | |
339 | ty::Float(f) => (Style::Float, f.bit_width()), | |
340 | _ => (Style::Unsupported, 0), | |
341 | }; | |
342 | ||
343 | let extend = |in_type, out_type| { | |
344 | let vector_type = bx.context.new_vector_type(out_type, 8); | |
345 | let vector = args[0].immediate(); | |
346 | let array_type = bx.context.new_array_type(None, in_type, 8); | |
347 | // TODO(antoyo): switch to using new_vector_access or __builtin_convertvector for vector casting. | |
348 | let array = bx.context.new_bitcast(None, vector, array_type); | |
349 | ||
350 | let cast_vec_element = |index| { | |
351 | let index = bx.context.new_rvalue_from_int(bx.int_type, index); | |
352 | bx.context.new_cast(None, bx.context.new_array_access(None, array, index).to_rvalue(), out_type) | |
353 | }; | |
354 | ||
355 | bx.context.new_rvalue_from_vector(None, vector_type, &[ | |
356 | cast_vec_element(0), | |
357 | cast_vec_element(1), | |
358 | cast_vec_element(2), | |
359 | cast_vec_element(3), | |
360 | cast_vec_element(4), | |
361 | cast_vec_element(5), | |
362 | cast_vec_element(6), | |
363 | cast_vec_element(7), | |
364 | ]) | |
365 | }; | |
366 | ||
367 | match (in_style, out_style) { | |
368 | (Style::Int(in_is_signed), Style::Int(_)) => { | |
369 | return Ok(match in_width.cmp(&out_width) { | |
370 | Ordering::Greater => bx.trunc(args[0].immediate(), llret_ty), | |
371 | Ordering::Equal => args[0].immediate(), | |
372 | Ordering::Less => { | |
373 | if in_is_signed { | |
374 | match (in_width, out_width) { | |
375 | // FIXME(antoyo): the function _mm_cvtepi8_epi16 should directly | |
376 | // call an intrinsic equivalent to __builtin_ia32_pmovsxbw128 so that | |
377 | // we can generate a call to it. | |
378 | (8, 16) => extend(bx.i8_type, bx.i16_type), | |
379 | (8, 32) => extend(bx.i8_type, bx.i32_type), | |
380 | (8, 64) => extend(bx.i8_type, bx.i64_type), | |
381 | (16, 32) => extend(bx.i16_type, bx.i32_type), | |
382 | (32, 64) => extend(bx.i32_type, bx.i64_type), | |
383 | (16, 64) => extend(bx.i16_type, bx.i64_type), | |
384 | _ => unimplemented!("in: {}, out: {}", in_width, out_width), | |
385 | } | |
386 | } else { | |
387 | match (in_width, out_width) { | |
388 | (8, 16) => extend(bx.u8_type, bx.u16_type), | |
389 | (8, 32) => extend(bx.u8_type, bx.u32_type), | |
390 | (8, 64) => extend(bx.u8_type, bx.u64_type), | |
391 | (16, 32) => extend(bx.u16_type, bx.u32_type), | |
392 | (16, 64) => extend(bx.u16_type, bx.u64_type), | |
393 | (32, 64) => extend(bx.u32_type, bx.u64_type), | |
394 | _ => unimplemented!("in: {}, out: {}", in_width, out_width), | |
395 | } | |
396 | } | |
397 | } | |
398 | }); | |
399 | } | |
400 | (Style::Int(_), Style::Float) => { | |
401 | // TODO: add support for internal functions in libgccjit to get access to IFN_VEC_CONVERT which is | |
402 | // doing like __builtin_convertvector? | |
403 | // Or maybe provide convert_vector as an API since it might not easy to get the | |
404 | // types of internal functions. | |
405 | unimplemented!(); | |
406 | } | |
407 | (Style::Float, Style::Int(_)) => { | |
408 | unimplemented!(); | |
409 | } | |
410 | (Style::Float, Style::Float) => { | |
411 | unimplemented!(); | |
412 | } | |
413 | _ => { /* Unsupported. Fallthrough. */ } | |
414 | } | |
415 | require!( | |
416 | false, | |
417 | "unsupported cast from `{}` with element `{}` to `{}` with element `{}`", | |
418 | in_ty, | |
419 | in_elem, | |
420 | ret_ty, | |
421 | out_elem | |
422 | ); | |
423 | } | |
424 | ||
c295e0f8 XL |
425 | macro_rules! arith_binary { |
426 | ($($name: ident: $($($p: ident),* => $call: ident),*;)*) => { | |
427 | $(if name == sym::$name { | |
428 | match in_elem.kind() { | |
429 | $($(ty::$p(_))|* => { | |
430 | return Ok(bx.$call(args[0].immediate(), args[1].immediate())) | |
431 | })* | |
432 | _ => {}, | |
433 | } | |
434 | require!(false, | |
435 | "unsupported operation on `{}` with element `{}`", | |
436 | in_ty, | |
437 | in_elem) | |
438 | })* | |
439 | } | |
440 | } | |
441 | ||
923072b8 FG |
442 | fn simd_simple_float_intrinsic<'gcc, 'tcx>( |
443 | name: Symbol, | |
444 | in_elem: Ty<'_>, | |
445 | in_ty: Ty<'_>, | |
446 | in_len: u64, | |
447 | bx: &mut Builder<'_, 'gcc, 'tcx>, | |
448 | span: Span, | |
449 | args: &[OperandRef<'tcx, RValue<'gcc>>], | |
450 | ) -> Result<RValue<'gcc>, ()> { | |
451 | macro_rules! emit_error { | |
452 | ($msg: tt, $($fmt: tt)*) => { | |
453 | span_invalid_monomorphization_error( | |
454 | bx.sess(), span, | |
455 | &format!(concat!("invalid monomorphization of `{}` intrinsic: ", $msg), | |
456 | name, $($fmt)*)); | |
457 | } | |
458 | } | |
459 | macro_rules! return_error { | |
460 | ($($fmt: tt)*) => { | |
461 | { | |
462 | emit_error!($($fmt)*); | |
463 | return Err(()); | |
464 | } | |
465 | } | |
466 | } | |
467 | ||
468 | let (elem_ty_str, elem_ty) = | |
469 | if let ty::Float(f) = in_elem.kind() { | |
470 | let elem_ty = bx.cx.type_float_from_ty(*f); | |
471 | match f.bit_width() { | |
472 | 32 => ("f32", elem_ty), | |
473 | 64 => ("f64", elem_ty), | |
474 | _ => { | |
475 | return_error!( | |
476 | "unsupported element type `{}` of floating-point vector `{}`", | |
477 | f.name_str(), | |
478 | in_ty | |
479 | ); | |
480 | } | |
481 | } | |
482 | } | |
483 | else { | |
484 | return_error!("`{}` is not a floating-point type", in_ty); | |
485 | }; | |
486 | ||
487 | let vec_ty = bx.cx.type_vector(elem_ty, in_len); | |
488 | ||
489 | let (intr_name, fn_ty) = | |
490 | match name { | |
491 | sym::simd_ceil => ("ceil", bx.type_func(&[vec_ty], vec_ty)), | |
492 | sym::simd_fabs => ("fabs", bx.type_func(&[vec_ty], vec_ty)), // TODO(antoyo): pand with 170141183420855150465331762880109871103 | |
493 | sym::simd_fcos => ("cos", bx.type_func(&[vec_ty], vec_ty)), | |
494 | sym::simd_fexp2 => ("exp2", bx.type_func(&[vec_ty], vec_ty)), | |
495 | sym::simd_fexp => ("exp", bx.type_func(&[vec_ty], vec_ty)), | |
496 | sym::simd_flog10 => ("log10", bx.type_func(&[vec_ty], vec_ty)), | |
497 | sym::simd_flog2 => ("log2", bx.type_func(&[vec_ty], vec_ty)), | |
498 | sym::simd_flog => ("log", bx.type_func(&[vec_ty], vec_ty)), | |
499 | sym::simd_floor => ("floor", bx.type_func(&[vec_ty], vec_ty)), | |
500 | sym::simd_fma => ("fma", bx.type_func(&[vec_ty, vec_ty, vec_ty], vec_ty)), | |
501 | sym::simd_fpowi => ("powi", bx.type_func(&[vec_ty, bx.type_i32()], vec_ty)), | |
502 | sym::simd_fpow => ("pow", bx.type_func(&[vec_ty, vec_ty], vec_ty)), | |
503 | sym::simd_fsin => ("sin", bx.type_func(&[vec_ty], vec_ty)), | |
504 | sym::simd_fsqrt => ("sqrt", bx.type_func(&[vec_ty], vec_ty)), | |
505 | sym::simd_round => ("round", bx.type_func(&[vec_ty], vec_ty)), | |
506 | sym::simd_trunc => ("trunc", bx.type_func(&[vec_ty], vec_ty)), | |
507 | _ => return_error!("unrecognized intrinsic `{}`", name), | |
508 | }; | |
509 | let llvm_name = &format!("llvm.{0}.v{1}{2}", intr_name, in_len, elem_ty_str); | |
510 | let function = intrinsic::llvm::intrinsic(llvm_name, &bx.cx); | |
511 | let function: RValue<'gcc> = unsafe { std::mem::transmute(function) }; | |
512 | let c = bx.call(fn_ty, function, &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(), None); | |
513 | Ok(c) | |
514 | } | |
515 | ||
516 | if std::matches!( | |
517 | name, | |
518 | sym::simd_ceil | |
519 | | sym::simd_fabs | |
520 | | sym::simd_fcos | |
521 | | sym::simd_fexp2 | |
522 | | sym::simd_fexp | |
523 | | sym::simd_flog10 | |
524 | | sym::simd_flog2 | |
525 | | sym::simd_flog | |
526 | | sym::simd_floor | |
527 | | sym::simd_fma | |
528 | | sym::simd_fpow | |
529 | | sym::simd_fpowi | |
530 | | sym::simd_fsin | |
531 | | sym::simd_fsqrt | |
532 | | sym::simd_round | |
533 | | sym::simd_trunc | |
534 | ) { | |
535 | return simd_simple_float_intrinsic(name, in_elem, in_ty, in_len, bx, span, args); | |
536 | } | |
537 | ||
c295e0f8 XL |
538 | arith_binary! { |
539 | simd_add: Uint, Int => add, Float => fadd; | |
540 | simd_sub: Uint, Int => sub, Float => fsub; | |
541 | simd_mul: Uint, Int => mul, Float => fmul; | |
542 | simd_div: Uint => udiv, Int => sdiv, Float => fdiv; | |
543 | simd_rem: Uint => urem, Int => srem, Float => frem; | |
544 | simd_shl: Uint, Int => shl; | |
545 | simd_shr: Uint => lshr, Int => ashr; | |
546 | simd_and: Uint, Int => and; | |
547 | simd_or: Uint, Int => or; // FIXME(antoyo): calling `or` might not work on vectors. | |
548 | simd_xor: Uint, Int => xor; | |
549 | } | |
550 | ||
5e7ed085 FG |
551 | macro_rules! arith_unary { |
552 | ($($name: ident: $($($p: ident),* => $call: ident),*;)*) => { | |
553 | $(if name == sym::$name { | |
554 | match in_elem.kind() { | |
555 | $($(ty::$p(_))|* => { | |
556 | return Ok(bx.$call(args[0].immediate())) | |
557 | })* | |
558 | _ => {}, | |
559 | } | |
560 | require!(false, | |
561 | "unsupported operation on `{}` with element `{}`", | |
562 | in_ty, | |
563 | in_elem) | |
564 | })* | |
565 | } | |
566 | } | |
567 | ||
568 | arith_unary! { | |
569 | simd_neg: Int => neg, Float => fneg; | |
570 | } | |
571 | ||
923072b8 FG |
572 | #[cfg(feature="master")] |
573 | if name == sym::simd_saturating_add || name == sym::simd_saturating_sub { | |
574 | let lhs = args[0].immediate(); | |
575 | let rhs = args[1].immediate(); | |
576 | let is_add = name == sym::simd_saturating_add; | |
577 | let ptr_bits = bx.tcx().data_layout.pointer_size.bits() as _; | |
578 | let (signed, elem_width, elem_ty) = match *in_elem.kind() { | |
579 | ty::Int(i) => (true, i.bit_width().unwrap_or(ptr_bits), bx.cx.type_int_from_ty(i)), | |
580 | ty::Uint(i) => (false, i.bit_width().unwrap_or(ptr_bits), bx.cx.type_uint_from_ty(i)), | |
581 | _ => { | |
582 | return_error!( | |
583 | "expected element type `{}` of vector type `{}` \ | |
584 | to be a signed or unsigned integer type", | |
585 | arg_tys[0].simd_size_and_type(bx.tcx()).1, | |
586 | arg_tys[0] | |
587 | ); | |
588 | } | |
589 | }; | |
590 | let builtin_name = | |
591 | match (signed, is_add, in_len, elem_width) { | |
592 | (true, true, 32, 8) => "__builtin_ia32_paddsb256", // TODO(antoyo): cast arguments to unsigned. | |
593 | (false, true, 32, 8) => "__builtin_ia32_paddusb256", | |
594 | (true, true, 16, 16) => "__builtin_ia32_paddsw256", | |
595 | (false, true, 16, 16) => "__builtin_ia32_paddusw256", | |
596 | (true, false, 16, 16) => "__builtin_ia32_psubsw256", | |
597 | (false, false, 16, 16) => "__builtin_ia32_psubusw256", | |
598 | (true, false, 32, 8) => "__builtin_ia32_psubsb256", | |
599 | (false, false, 32, 8) => "__builtin_ia32_psubusb256", | |
600 | _ => unimplemented!("signed: {}, is_add: {}, in_len: {}, elem_width: {}", signed, is_add, in_len, elem_width), | |
601 | }; | |
602 | let vec_ty = bx.cx.type_vector(elem_ty, in_len as u64); | |
603 | ||
604 | let func = bx.context.get_target_builtin_function(builtin_name); | |
605 | let param1_type = func.get_param(0).to_rvalue().get_type(); | |
606 | let param2_type = func.get_param(1).to_rvalue().get_type(); | |
607 | let lhs = bx.cx.bitcast_if_needed(lhs, param1_type); | |
608 | let rhs = bx.cx.bitcast_if_needed(rhs, param2_type); | |
609 | let result = bx.context.new_call(None, func, &[lhs, rhs]); | |
610 | // TODO(antoyo): perhaps use __builtin_convertvector for vector casting. | |
611 | return Ok(bx.context.new_bitcast(None, result, vec_ty)); | |
612 | } | |
613 | ||
614 | macro_rules! arith_red { | |
615 | ($name:ident : $vec_op:expr, $float_reduce:ident, $ordered:expr, $op:ident, | |
616 | $identity:expr) => { | |
617 | if name == sym::$name { | |
618 | require!( | |
619 | ret_ty == in_elem, | |
620 | "expected return type `{}` (element of input `{}`), found `{}`", | |
621 | in_elem, | |
622 | in_ty, | |
623 | ret_ty | |
624 | ); | |
625 | return match in_elem.kind() { | |
626 | ty::Int(_) | ty::Uint(_) => { | |
627 | let r = bx.vector_reduce_op(args[0].immediate(), $vec_op); | |
628 | if $ordered { | |
629 | // if overflow occurs, the result is the | |
630 | // mathematical result modulo 2^n: | |
631 | Ok(bx.$op(args[1].immediate(), r)) | |
632 | } | |
633 | else { | |
634 | Ok(bx.vector_reduce_op(args[0].immediate(), $vec_op)) | |
635 | } | |
636 | } | |
637 | ty::Float(_) => { | |
638 | if $ordered { | |
639 | // ordered arithmetic reductions take an accumulator | |
640 | let acc = args[1].immediate(); | |
641 | Ok(bx.$float_reduce(acc, args[0].immediate())) | |
642 | } | |
643 | else { | |
644 | Ok(bx.vector_reduce_op(args[0].immediate(), $vec_op)) | |
645 | } | |
646 | } | |
647 | _ => return_error!( | |
648 | "unsupported {} from `{}` with element `{}` to `{}`", | |
649 | sym::$name, | |
650 | in_ty, | |
651 | in_elem, | |
652 | ret_ty | |
653 | ), | |
654 | }; | |
655 | } | |
656 | }; | |
657 | } | |
658 | ||
659 | arith_red!( | |
660 | simd_reduce_add_unordered: BinaryOp::Plus, | |
661 | vector_reduce_fadd_fast, | |
662 | false, | |
663 | add, | |
664 | 0.0 // TODO: Use this argument. | |
665 | ); | |
666 | arith_red!( | |
667 | simd_reduce_mul_unordered: BinaryOp::Mult, | |
668 | vector_reduce_fmul_fast, | |
669 | false, | |
670 | mul, | |
671 | 1.0 | |
672 | ); | |
673 | ||
674 | macro_rules! minmax_red { | |
675 | ($name:ident: $reduction:ident) => { | |
676 | if name == sym::$name { | |
677 | require!( | |
678 | ret_ty == in_elem, | |
679 | "expected return type `{}` (element of input `{}`), found `{}`", | |
680 | in_elem, | |
681 | in_ty, | |
682 | ret_ty | |
683 | ); | |
684 | return match in_elem.kind() { | |
685 | ty::Int(_) | ty::Uint(_) | ty::Float(_) => Ok(bx.$reduction(args[0].immediate())), | |
686 | _ => return_error!( | |
687 | "unsupported {} from `{}` with element `{}` to `{}`", | |
688 | sym::$name, | |
689 | in_ty, | |
690 | in_elem, | |
691 | ret_ty | |
692 | ), | |
693 | }; | |
694 | } | |
695 | }; | |
696 | } | |
697 | ||
698 | minmax_red!(simd_reduce_min: vector_reduce_min); | |
699 | minmax_red!(simd_reduce_max: vector_reduce_max); | |
700 | ||
701 | macro_rules! bitwise_red { | |
702 | ($name:ident : $op:expr, $boolean:expr) => { | |
703 | if name == sym::$name { | |
704 | let input = if !$boolean { | |
705 | require!( | |
706 | ret_ty == in_elem, | |
707 | "expected return type `{}` (element of input `{}`), found `{}`", | |
708 | in_elem, | |
709 | in_ty, | |
710 | ret_ty | |
711 | ); | |
712 | args[0].immediate() | |
713 | } else { | |
714 | match in_elem.kind() { | |
715 | ty::Int(_) | ty::Uint(_) => {} | |
716 | _ => return_error!( | |
717 | "unsupported {} from `{}` with element `{}` to `{}`", | |
718 | sym::$name, | |
719 | in_ty, | |
720 | in_elem, | |
721 | ret_ty | |
722 | ), | |
723 | } | |
724 | ||
725 | // boolean reductions operate on vectors of i1s: | |
726 | let i1 = bx.type_i1(); | |
727 | let i1xn = bx.type_vector(i1, in_len as u64); | |
728 | bx.trunc(args[0].immediate(), i1xn) | |
729 | }; | |
730 | return match in_elem.kind() { | |
731 | ty::Int(_) | ty::Uint(_) => { | |
732 | let r = bx.vector_reduce_op(input, $op); | |
733 | Ok(if !$boolean { r } else { bx.zext(r, bx.type_bool()) }) | |
734 | } | |
735 | _ => return_error!( | |
736 | "unsupported {} from `{}` with element `{}` to `{}`", | |
737 | sym::$name, | |
738 | in_ty, | |
739 | in_elem, | |
740 | ret_ty | |
741 | ), | |
742 | }; | |
743 | } | |
744 | }; | |
745 | } | |
746 | ||
747 | bitwise_red!(simd_reduce_and: BinaryOp::BitwiseAnd, false); | |
748 | bitwise_red!(simd_reduce_or: BinaryOp::BitwiseOr, false); | |
749 | ||
c295e0f8 XL |
750 | unimplemented!("simd {}", name); |
751 | } |