]> git.proxmox.com Git - rustc.git/blame - compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
projects / rustc.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
New upstream version 1.65.0+dfsg1
[rustc.git] / compiler / rustc_codegen_cranelift / src / intrinsics / simd.rs
CommitLineData
29967ef6
XL		 1//! Codegen `extern "platform-intrinsic"` intrinsics.
2
5e7ed085
FG		 3use rustc_middle::ty::subst::SubstsRef;
4use rustc_span::Symbol;
5
29967ef6
XL		 6use super::*;
7use crate::prelude::*;
8
5e7ed085
FG		 9fn report_simd_type_validation_error(
10 fx: &mut FunctionCx<'_, '_, '_>,
11 intrinsic: Symbol,
12 span: Span,
13 ty: Ty<'_>,
14) {
15 fx.tcx.sess.span_err(span, &format!("invalid monomorphization of `{}` intrinsic: expected SIMD input type, found non-SIMD `{}`", intrinsic, ty));
16 // Prevent verifier error
f2b60f7d 17 fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
5e7ed085
FG		 18}
19
29967ef6 20pub(super) fn codegen_simd_intrinsic_call<'tcx>(
6a06907d 21 fx: &mut FunctionCx<'_, '_, 'tcx>,
5e7ed085
FG		 22 intrinsic: Symbol,
23 _substs: SubstsRef<'tcx>,
29967ef6
XL		 24 args: &[mir::Operand<'tcx>],
25 ret: CPlace<'tcx>,
26 span: Span,
27) {
064997fb
FG		 28 match intrinsic {
29 sym::simd_cast => {
30 intrinsic_args!(fx, args => (a); intrinsic);
29967ef6 31
5e7ed085
FG		 32 if !a.layout().ty.is_simd() {
33 report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
34 return;
35 }
36
37 simd_for_each_lane(fx, a, ret, &|fx, lane_ty, ret_lane_ty, lane| {
38 let ret_lane_clif_ty = fx.clif_type(ret_lane_ty).unwrap();
29967ef6 39
5e7ed085
FG		 40 let from_signed = type_sign(lane_ty);
41 let to_signed = type_sign(ret_lane_ty);
29967ef6 42
5e7ed085 43 clif_int_or_float_cast(fx, lane, from_signed, ret_lane_clif_ty, to_signed)
29967ef6 44 });
064997fb
FG		 45 }
46
47 sym::simd_eq | sym::simd_ne | sym::simd_lt | sym::simd_le | sym::simd_gt | sym::simd_ge => {
48 intrinsic_args!(fx, args => (x, y); intrinsic);
29967ef6 49
5e7ed085
FG		 50 if !x.layout().ty.is_simd() {
51 report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty);
52 return;
53 }
54
55 // FIXME use vector instructions when possible
56 simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, res_lane_ty, x_lane, y_lane| {
57 let res_lane = match (lane_ty.kind(), intrinsic) {
58 (ty::Uint(_), sym::simd_eq) => fx.bcx.ins().icmp(IntCC::Equal, x_lane, y_lane),
064997fb
FG		 59 (ty::Uint(_), sym::simd_ne) => {
60 fx.bcx.ins().icmp(IntCC::NotEqual, x_lane, y_lane)
61 }
5e7ed085
FG		 62 (ty::Uint(_), sym::simd_lt) => {
63 fx.bcx.ins().icmp(IntCC::UnsignedLessThan, x_lane, y_lane)
64 }
65 (ty::Uint(_), sym::simd_le) => {
66 fx.bcx.ins().icmp(IntCC::UnsignedLessThanOrEqual, x_lane, y_lane)
67 }
68 (ty::Uint(_), sym::simd_gt) => {
69 fx.bcx.ins().icmp(IntCC::UnsignedGreaterThan, x_lane, y_lane)
70 }
71 (ty::Uint(_), sym::simd_ge) => {
72 fx.bcx.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, x_lane, y_lane)
73 }
74
75 (ty::Int(_), sym::simd_eq) => fx.bcx.ins().icmp(IntCC::Equal, x_lane, y_lane),
064997fb
FG		 76 (ty::Int(_), sym::simd_ne) => {
77 fx.bcx.ins().icmp(IntCC::NotEqual, x_lane, y_lane)
78 }
79 (ty::Int(_), sym::simd_lt) => {
80 fx.bcx.ins().icmp(IntCC::SignedLessThan, x_lane, y_lane)
81 }
5e7ed085
FG		 82 (ty::Int(_), sym::simd_le) => {
83 fx.bcx.ins().icmp(IntCC::SignedLessThanOrEqual, x_lane, y_lane)
84 }
85 (ty::Int(_), sym::simd_gt) => {
86 fx.bcx.ins().icmp(IntCC::SignedGreaterThan, x_lane, y_lane)
87 }
88 (ty::Int(_), sym::simd_ge) => {
89 fx.bcx.ins().icmp(IntCC::SignedGreaterThanOrEqual, x_lane, y_lane)
90 }
91
064997fb
FG		 92 (ty::Float(_), sym::simd_eq) => {
93 fx.bcx.ins().fcmp(FloatCC::Equal, x_lane, y_lane)
94 }
95 (ty::Float(_), sym::simd_ne) => {
96 fx.bcx.ins().fcmp(FloatCC::NotEqual, x_lane, y_lane)
97 }
98 (ty::Float(_), sym::simd_lt) => {
99 fx.bcx.ins().fcmp(FloatCC::LessThan, x_lane, y_lane)
100 }
5e7ed085
FG		 101 (ty::Float(_), sym::simd_le) => {
102 fx.bcx.ins().fcmp(FloatCC::LessThanOrEqual, x_lane, y_lane)
103 }
064997fb
FG		 104 (ty::Float(_), sym::simd_gt) => {
105 fx.bcx.ins().fcmp(FloatCC::GreaterThan, x_lane, y_lane)
106 }
5e7ed085
FG		 107 (ty::Float(_), sym::simd_ge) => {
108 fx.bcx.ins().fcmp(FloatCC::GreaterThanOrEqual, x_lane, y_lane)
109 }
110
111 _ => unreachable!(),
112 };
113
114 let ty = fx.clif_type(res_lane_ty).unwrap();
115
116 let res_lane = fx.bcx.ins().bint(ty, res_lane);
117 fx.bcx.ins().ineg(res_lane)
118 });
064997fb 119 }
29967ef6
XL		 120
121 // simd_shuffle32<T, U>(x: T, y: T, idx: [u32; 32]) -> U
064997fb
FG		 122 _ if intrinsic.as_str().starts_with("simd_shuffle") => {
123 let (x, y, idx) = match args {
124 [x, y, idx] => (x, y, idx),
125 _ => {
126 bug!("wrong number of args for intrinsic {intrinsic}");
127 }
128 };
129 let x = codegen_operand(fx, x);
130 let y = codegen_operand(fx, y);
131
5e7ed085
FG		 132 if !x.layout().ty.is_simd() {
133 report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty);
134 return;
135 }
29967ef6 136
a2a8927a
XL		 137 // If this intrinsic is the older "simd_shuffleN" form, simply parse the integer.
138 // If there is no suffix, use the index array length.
139 let n: u16 = if intrinsic == sym::simd_shuffle {
140 // Make sure this is actually an array, since typeck only checks the length-suffixed
141 // version of this intrinsic.
142 let idx_ty = fx.monomorphize(idx.ty(fx.mir, fx.tcx));
143 match idx_ty.kind() {
064997fb
FG		 144 ty::Array(ty, len) if matches!(ty.kind(), ty::Uint(ty::UintTy::U32)) => len
145 .try_eval_usize(fx.tcx, ty::ParamEnv::reveal_all())
146 .unwrap_or_else(|| {
a2a8927a 147 span_bug!(span, "could not evaluate shuffle index array length")
064997fb
FG		 148 })
149 .try_into()
150 .unwrap(),
a2a8927a
XL		 151 _ => {
152 fx.tcx.sess.span_err(
153 span,
154 &format!(
155 "simd_shuffle index must be an array of `u32`, got `{}`",
156 idx_ty,
157 ),
158 );
159 // Prevent verifier error
f2b60f7d 160 fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
a2a8927a
XL		 161 return;
162 }
163 }
164 } else {
165 intrinsic.as_str()["simd_shuffle".len()..].parse().unwrap()
166 };
29967ef6
XL		 167
168 assert_eq!(x.layout(), y.layout());
169 let layout = x.layout();
170
5869c6ff
XL		 171 let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
172 let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
29967ef6 173
5869c6ff
XL		 174 assert_eq!(lane_ty, ret_lane_ty);
175 assert_eq!(u64::from(n), ret_lane_count);
29967ef6
XL		 176
177 let total_len = lane_count * 2;
178
179 let indexes = {
180 use rustc_middle::mir::interpret::*;
064997fb
FG		 181 let idx_const = crate::constant::mir_operand_get_const_val(fx, idx)
182 .expect("simd_shuffle* idx not const");
29967ef6 183
6a06907d
XL		 184 let idx_bytes = match idx_const {
185 ConstValue::ByRef { alloc, offset } => {
064997fb
FG		 186 let size = Size::from_bytes(
187 4 * ret_lane_count, /* size_of([u32; ret_lane_count]) */
188 );
f2b60f7d
FG		 189 alloc
190 .inner()
191 .get_bytes_strip_provenance(fx, alloc_range(offset, size))
192 .unwrap()
29967ef6
XL		 193 }
194 _ => unreachable!("{:?}", idx_const),
195 };
196
064997fb
FG		 197 (0..ret_lane_count)
198 .map(|i| {
199 let i = usize::try_from(i).unwrap();
200 let idx = rustc_middle::mir::interpret::read_target_uint(
201 fx.tcx.data_layout.endian,
202 &idx_bytes[4 * i..4 * i + 4],
203 )
204 .expect("read_target_uint");
205 u16::try_from(idx).expect("try_from u32")
206 })
207 .collect::<Vec<u16>>()
29967ef6
XL		 208 };
209
210 for &idx in &indexes {
5869c6ff 211 assert!(u64::from(idx) < total_len, "idx {} out of range 0..{}", idx, total_len);
29967ef6
XL		 212 }
213
214 for (out_idx, in_idx) in indexes.into_iter().enumerate() {
5869c6ff 215 let in_lane = if u64::from(in_idx) < lane_count {
94222f64 216 x.value_lane(fx, in_idx.into())
29967ef6 217 } else {
94222f64 218 y.value_lane(fx, u64::from(in_idx) - lane_count)
29967ef6 219 };
94222f64 220 let out_lane = ret.place_lane(fx, u64::try_from(out_idx).unwrap());
29967ef6
XL		 221 out_lane.write_cvalue(fx, in_lane);
222 }
064997fb
FG		 223 }
224
225 sym::simd_insert => {
226 let (base, idx, val) = match args {
227 [base, idx, val] => (base, idx, val),
228 _ => {
229 bug!("wrong number of args for intrinsic {intrinsic}");
230 }
231 };
232 let base = codegen_operand(fx, base);
233 let val = codegen_operand(fx, val);
29967ef6 234
29967ef6 235 // FIXME validate
064997fb
FG		 236 let idx_const = if let Some(idx_const) =
237 crate::constant::mir_operand_get_const_val(fx, idx)
238 {
29967ef6
XL		 239 idx_const
240 } else {
064997fb 241 fx.tcx.sess.span_fatal(span, "Index argument for `simd_insert` is not a constant");
29967ef6
XL		 242 };
243
064997fb
FG		 244 let idx = idx_const
245 .try_to_bits(Size::from_bytes(4 /* u32*/))
246 .unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const));
5869c6ff 247 let (lane_count, _lane_ty) = base.layout().ty.simd_size_and_type(fx.tcx);
29967ef6 248 if idx >= lane_count.into() {
064997fb
FG		 249 fx.tcx.sess.span_fatal(
250 fx.mir.span,
251 &format!("[simd_insert] idx {} >= lane_count {}", idx, lane_count),
252 );
29967ef6
XL		 253 }
254
255 ret.write_cvalue(fx, base);
256 let ret_lane = ret.place_field(fx, mir::Field::new(idx.try_into().unwrap()));
257 ret_lane.write_cvalue(fx, val);
064997fb
FG		 258 }
259
260 sym::simd_extract => {
261 let (v, idx) = match args {
262 [v, idx] => (v, idx),
263 _ => {
264 bug!("wrong number of args for intrinsic {intrinsic}");
265 }
266 };
267 let v = codegen_operand(fx, v);
29967ef6 268
5e7ed085
FG		 269 if !v.layout().ty.is_simd() {
270 report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
271 return;
272 }
273
064997fb
FG		 274 let idx_const = if let Some(idx_const) =
275 crate::constant::mir_operand_get_const_val(fx, idx)
276 {
29967ef6
XL		 277 idx_const
278 } else {
064997fb 279 fx.tcx.sess.span_warn(span, "Index argument for `simd_extract` is not a constant");
f2b60f7d
FG		 280 let trap_block = fx.bcx.create_block();
281 let dummy_block = fx.bcx.create_block();
282 let true_ = fx.bcx.ins().iconst(types::I8, 1);
283 fx.bcx.ins().brnz(true_, trap_block, &[]);
284 fx.bcx.ins().jump(dummy_block, &[]);
285 fx.bcx.switch_to_block(trap_block);
286 crate::trap::trap_unimplemented(
fc512014 287 fx,
fc512014
XL		 288 "Index argument for `simd_extract` is not a constant",
289 );
f2b60f7d 290 fx.bcx.switch_to_block(dummy_block);
fc512014 291 return;
29967ef6
XL		 292 };
293
064997fb
FG		 294 let idx = idx_const
295 .try_to_bits(Size::from_bytes(4 /* u32*/))
296 .unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const));
5869c6ff 297 let (lane_count, _lane_ty) = v.layout().ty.simd_size_and_type(fx.tcx);
29967ef6 298 if idx >= lane_count.into() {
064997fb
FG		 299 fx.tcx.sess.span_fatal(
300 fx.mir.span,
301 &format!("[simd_extract] idx {} >= lane_count {}", idx, lane_count),
302 );
29967ef6
XL		 303 }
304
94222f64 305 let ret_lane = v.value_lane(fx, idx.try_into().unwrap());
29967ef6 306 ret.write_cvalue(fx, ret_lane);
064997fb
FG		 307 }
308
309 sym::simd_neg => {
310 intrinsic_args!(fx, args => (a); intrinsic);
29967ef6 311
5e7ed085
FG		 312 if !a.layout().ty.is_simd() {
313 report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
314 return;
315 }
316
064997fb
FG		 317 simd_for_each_lane(
318 fx,
319 a,
320 ret,
321 &|fx, lane_ty, _ret_lane_ty, lane| match lane_ty.kind() {
94222f64
XL		 322 ty::Int(_) => fx.bcx.ins().ineg(lane),
323 ty::Float(_) => fx.bcx.ins().fneg(lane),
324 _ => unreachable!(),
064997fb
FG		 325 },
326 );
327 }
328
329 sym::simd_add
330 | sym::simd_sub
331 | sym::simd_mul
332 | sym::simd_div
333 | sym::simd_rem
334 | sym::simd_shl
335 | sym::simd_shr
336 | sym::simd_and
337 | sym::simd_or
338 | sym::simd_xor => {
339 intrinsic_args!(fx, args => (x, y); intrinsic);
94222f64 340
5e7ed085 341 // FIXME use vector instructions when possible
064997fb
FG		 342 simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, _ret_lane_ty, x_lane, y_lane| {
343 match (lane_ty.kind(), intrinsic) {
344 (ty::Uint(_), sym::simd_add) => fx.bcx.ins().iadd(x_lane, y_lane),
345 (ty::Uint(_), sym::simd_sub) => fx.bcx.ins().isub(x_lane, y_lane),
346 (ty::Uint(_), sym::simd_mul) => fx.bcx.ins().imul(x_lane, y_lane),
347 (ty::Uint(_), sym::simd_div) => fx.bcx.ins().udiv(x_lane, y_lane),
348 (ty::Uint(_), sym::simd_rem) => fx.bcx.ins().urem(x_lane, y_lane),
349
350 (ty::Int(_), sym::simd_add) => fx.bcx.ins().iadd(x_lane, y_lane),
351 (ty::Int(_), sym::simd_sub) => fx.bcx.ins().isub(x_lane, y_lane),
352 (ty::Int(_), sym::simd_mul) => fx.bcx.ins().imul(x_lane, y_lane),
353 (ty::Int(_), sym::simd_div) => fx.bcx.ins().sdiv(x_lane, y_lane),
354 (ty::Int(_), sym::simd_rem) => fx.bcx.ins().srem(x_lane, y_lane),
355
356 (ty::Float(_), sym::simd_add) => fx.bcx.ins().fadd(x_lane, y_lane),
357 (ty::Float(_), sym::simd_sub) => fx.bcx.ins().fsub(x_lane, y_lane),
358 (ty::Float(_), sym::simd_mul) => fx.bcx.ins().fmul(x_lane, y_lane),
359 (ty::Float(_), sym::simd_div) => fx.bcx.ins().fdiv(x_lane, y_lane),
360 (ty::Float(FloatTy::F32), sym::simd_rem) => fx.lib_call(
361 "fmodf",
362 vec![AbiParam::new(types::F32), AbiParam::new(types::F32)],
363 vec![AbiParam::new(types::F32)],
364 &[x_lane, y_lane],
365 )[0],
366 (ty::Float(FloatTy::F64), sym::simd_rem) => fx.lib_call(
367 "fmod",
368 vec![AbiParam::new(types::F64), AbiParam::new(types::F64)],
369 vec![AbiParam::new(types::F64)],
370 &[x_lane, y_lane],
371 )[0],
372
373 (ty::Uint(_), sym::simd_shl) => fx.bcx.ins().ishl(x_lane, y_lane),
374 (ty::Uint(_), sym::simd_shr) => fx.bcx.ins().ushr(x_lane, y_lane),
375 (ty::Uint(_), sym::simd_and) => fx.bcx.ins().band(x_lane, y_lane),
376 (ty::Uint(_), sym::simd_or) => fx.bcx.ins().bor(x_lane, y_lane),
377 (ty::Uint(_), sym::simd_xor) => fx.bcx.ins().bxor(x_lane, y_lane),
378
379 (ty::Int(_), sym::simd_shl) => fx.bcx.ins().ishl(x_lane, y_lane),
380 (ty::Int(_), sym::simd_shr) => fx.bcx.ins().sshr(x_lane, y_lane),
381 (ty::Int(_), sym::simd_and) => fx.bcx.ins().band(x_lane, y_lane),
382 (ty::Int(_), sym::simd_or) => fx.bcx.ins().bor(x_lane, y_lane),
383 (ty::Int(_), sym::simd_xor) => fx.bcx.ins().bxor(x_lane, y_lane),
384
385 _ => unreachable!(),
386 }
94222f64 387 });
064997fb
FG		 388 }
389
390 sym::simd_fma => {
391 intrinsic_args!(fx, args => (a, b, c); intrinsic);
94222f64 392
5e7ed085
FG		 393 if !a.layout().ty.is_simd() {
394 report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
395 return;
396 }
29967ef6
XL		 397 assert_eq!(a.layout(), b.layout());
398 assert_eq!(a.layout(), c.layout());
04454e1e 399 assert_eq!(a.layout(), ret.layout());
29967ef6 400
04454e1e
FG		 401 let layout = a.layout();
402 let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
f2b60f7d 403 let res_lane_layout = fx.layout_of(lane_ty);
29967ef6
XL		 404
405 for lane in 0..lane_count {
f2b60f7d
FG		 406 let a_lane = a.value_lane(fx, lane).load_scalar(fx);
407 let b_lane = b.value_lane(fx, lane).load_scalar(fx);
408 let c_lane = c.value_lane(fx, lane).load_scalar(fx);
29967ef6 409
f2b60f7d
FG		 410 let res_lane = fx.bcx.ins().fma(a_lane, b_lane, c_lane);
411 let res_lane = CValue::by_val(res_lane, res_lane_layout);
29967ef6 412
94222f64 413 ret.place_lane(fx, lane).write_cvalue(fx, res_lane);
29967ef6 414 }
064997fb
FG		 415 }
416
417 sym::simd_fmin | sym::simd_fmax => {
418 intrinsic_args!(fx, args => (x, y); intrinsic);
29967ef6 419
5e7ed085
FG		 420 if !x.layout().ty.is_simd() {
421 report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty);
422 return;
423 }
424
425 // FIXME use vector instructions when possible
426 simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, _ret_lane_ty, x_lane, y_lane| {
427 match lane_ty.kind() {
064997fb 428 ty::Float(_) => {}
5e7ed085
FG		 429 _ => unreachable!("{:?}", lane_ty),
430 }
431 match intrinsic {
04454e1e
FG		 432 sym::simd_fmin => crate::num::codegen_float_min(fx, x_lane, y_lane),
433 sym::simd_fmax => crate::num::codegen_float_max(fx, x_lane, y_lane),
5e7ed085
FG		 434 _ => unreachable!(),
435 }
436 });
064997fb
FG		 437 }
438
439 sym::simd_round => {
440 intrinsic_args!(fx, args => (a); intrinsic);
29967ef6 441
5e7ed085
FG		 442 if !a.layout().ty.is_simd() {
443 report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
444 return;
445 }
446
064997fb
FG		 447 simd_for_each_lane(
448 fx,
449 a,
450 ret,
451 &|fx, lane_ty, _ret_lane_ty, lane| match lane_ty.kind() {
94222f64
XL		 452 ty::Float(FloatTy::F32) => fx.lib_call(
453 "roundf",
454 vec![AbiParam::new(types::F32)],
455 vec![AbiParam::new(types::F32)],
456 &[lane],
457 )[0],
458 ty::Float(FloatTy::F64) => fx.lib_call(
459 "round",
460 vec![AbiParam::new(types::F64)],
461 vec![AbiParam::new(types::F64)],
462 &[lane],
463 )[0],
5e7ed085 464 _ => unreachable!("{:?}", lane_ty),
064997fb
FG		 465 },
466 );
467 }
468
469 sym::simd_fabs | sym::simd_fsqrt | sym::simd_ceil | sym::simd_floor | sym::simd_trunc => {
470 intrinsic_args!(fx, args => (a); intrinsic);
5e7ed085 471
5e7ed085
FG		 472 if !a.layout().ty.is_simd() {
473 report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
474 return;
475 }
476
477 simd_for_each_lane(fx, a, ret, &|fx, lane_ty, _ret_lane_ty, lane| {
478 match lane_ty.kind() {
064997fb 479 ty::Float(_) => {}
5e7ed085
FG		 480 _ => unreachable!("{:?}", lane_ty),
481 }
482 match intrinsic {
483 sym::simd_fabs => fx.bcx.ins().fabs(lane),
484 sym::simd_fsqrt => fx.bcx.ins().sqrt(lane),
485 sym::simd_ceil => fx.bcx.ins().ceil(lane),
486 sym::simd_floor => fx.bcx.ins().floor(lane),
487 sym::simd_trunc => fx.bcx.ins().trunc(lane),
488 _ => unreachable!(),
489 }
94222f64 490 });
064997fb
FG		 491 }
492
493 sym::simd_reduce_add_ordered | sym::simd_reduce_add_unordered => {
494 intrinsic_args!(fx, args => (v, acc); intrinsic);
495 let acc = acc.load_scalar(fx);
94222f64 496
5e7ed085
FG		 497 // FIXME there must be no acc param for integer vectors
498 if !v.layout().ty.is_simd() {
499 report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
500 return;
501 }
502
503 simd_reduce(fx, v, Some(acc), ret, &|fx, lane_ty, a, b| {
504 if lane_ty.is_floating_point() {
fc512014
XL		 505 fx.bcx.ins().fadd(a, b)
506 } else {
507 fx.bcx.ins().iadd(a, b)
508 }
509 });
064997fb
FG		 510 }
511
512 sym::simd_reduce_mul_ordered | sym::simd_reduce_mul_unordered => {
513 intrinsic_args!(fx, args => (v, acc); intrinsic);
514 let acc = acc.load_scalar(fx);
fc512014 515
5e7ed085
FG		 516 // FIXME there must be no acc param for integer vectors
517 if !v.layout().ty.is_simd() {
518 report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
519 return;
520 }
521
522 simd_reduce(fx, v, Some(acc), ret, &|fx, lane_ty, a, b| {
523 if lane_ty.is_floating_point() {
fc512014
XL		 524 fx.bcx.ins().fmul(a, b)
525 } else {
526 fx.bcx.ins().imul(a, b)
527 }
528 });
064997fb
FG		 529 }
530
531 sym::simd_reduce_all => {
532 intrinsic_args!(fx, args => (v); intrinsic);
fc512014 533
5e7ed085
FG		 534 if !v.layout().ty.is_simd() {
535 report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
536 return;
537 }
538
539 simd_reduce_bool(fx, v, ret, &|fx, a, b| fx.bcx.ins().band(a, b));
064997fb
FG		 540 }
541
542 sym::simd_reduce_any => {
543 intrinsic_args!(fx, args => (v); intrinsic);
fc512014 544
5e7ed085
FG		 545 if !v.layout().ty.is_simd() {
546 report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
547 return;
548 }
549
550 simd_reduce_bool(fx, v, ret, &|fx, a, b| fx.bcx.ins().bor(a, b));
064997fb
FG		 551 }
552
553 sym::simd_reduce_and => {
554 intrinsic_args!(fx, args => (v); intrinsic);
fc512014 555
5e7ed085
FG		 556 if !v.layout().ty.is_simd() {
557 report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
558 return;
559 }
560
561 simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| fx.bcx.ins().band(a, b));
064997fb
FG		 562 }
563
564 sym::simd_reduce_or => {
565 intrinsic_args!(fx, args => (v); intrinsic);
94222f64 566
5e7ed085
FG		 567 if !v.layout().ty.is_simd() {
568 report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
569 return;
570 }
571
572 simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| fx.bcx.ins().bor(a, b));
064997fb
FG		 573 }
574
575 sym::simd_reduce_xor => {
576 intrinsic_args!(fx, args => (v); intrinsic);
94222f64 577
5e7ed085
FG		 578 if !v.layout().ty.is_simd() {
579 report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
580 return;
581 }
582
583 simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| fx.bcx.ins().bxor(a, b));
064997fb
FG		 584 }
585
586 sym::simd_reduce_min => {
587 intrinsic_args!(fx, args => (v); intrinsic);
94222f64 588
5e7ed085
FG		 589 if !v.layout().ty.is_simd() {
590 report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
591 return;
592 }
593
594 simd_reduce(fx, v, None, ret, &|fx, ty, a, b| {
595 let lt = match ty.kind() {
a2a8927a
XL		 596 ty::Int(_) => fx.bcx.ins().icmp(IntCC::SignedLessThan, a, b),
597 ty::Uint(_) => fx.bcx.ins().icmp(IntCC::UnsignedLessThan, a, b),
04454e1e 598 ty::Float(_) => return crate::num::codegen_float_min(fx, a, b),
a2a8927a
XL		 599 _ => unreachable!(),
600 };
94222f64
XL		 601 fx.bcx.ins().select(lt, a, b)
602 });
064997fb
FG		 603 }
604
605 sym::simd_reduce_max => {
606 intrinsic_args!(fx, args => (v); intrinsic);
94222f64 607
5e7ed085
FG		 608 if !v.layout().ty.is_simd() {
609 report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
610 return;
611 }
612
613 simd_reduce(fx, v, None, ret, &|fx, ty, a, b| {
614 let gt = match ty.kind() {
a2a8927a
XL		 615 ty::Int(_) => fx.bcx.ins().icmp(IntCC::SignedGreaterThan, a, b),
616 ty::Uint(_) => fx.bcx.ins().icmp(IntCC::UnsignedGreaterThan, a, b),
04454e1e 617 ty::Float(_) => return crate::num::codegen_float_max(fx, a, b),
a2a8927a
XL		 618 _ => unreachable!(),
619 };
94222f64
XL		 620 fx.bcx.ins().select(gt, a, b)
621 });
064997fb
FG		 622 }
623
624 sym::simd_select => {
625 intrinsic_args!(fx, args => (m, a, b); intrinsic);
94222f64 626
5e7ed085
FG		 627 if !m.layout().ty.is_simd() {
628 report_simd_type_validation_error(fx, intrinsic, span, m.layout().ty);
629 return;
630 }
631 if !a.layout().ty.is_simd() {
632 report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
633 return;
634 }
94222f64
XL		 635 assert_eq!(a.layout(), b.layout());
636
637 let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx);
638 let lane_layout = fx.layout_of(lane_ty);
639
640 for lane in 0..lane_count {
641 let m_lane = m.value_lane(fx, lane).load_scalar(fx);
642 let a_lane = a.value_lane(fx, lane).load_scalar(fx);
643 let b_lane = b.value_lane(fx, lane).load_scalar(fx);
644
645 let m_lane = fx.bcx.ins().icmp_imm(IntCC::Equal, m_lane, 0);
064997fb
FG		 646 let res_lane =
647 CValue::by_val(fx.bcx.ins().select(m_lane, b_lane, a_lane), lane_layout);
94222f64
XL		 648
649 ret.place_lane(fx, lane).write_cvalue(fx, res_lane);
650 }
064997fb 651 }
94222f64
XL		 652
653 // simd_saturating_*
29967ef6 654 // simd_bitmask
94222f64
XL		 655 // simd_scatter
656 // simd_gather
064997fb
FG		 657 _ => {
658 fx.tcx.sess.span_fatal(span, &format!("Unknown SIMD intrinsic {}", intrinsic));
659 }
29967ef6
XL		 660 }
661}
backport for Debian buster