[rustc.git] / compiler / rustc_codegen_cranelift / src / intrinsics / simd.rs

//! Codegen `extern "platform-intrinsic"` intrinsics.

use super::*;
use crate::prelude::*;

pub(super) fn codegen_simd_intrinsic_call<'tcx>(
    fx: &mut FunctionCx<'_, '_, 'tcx>,
    instance: Instance<'tcx>,
    args: &[mir::Operand<'tcx>],
    ret: CPlace<'tcx>,
    span: Span,
) {
    let def_id = instance.def_id();
    let substs = instance.substs;

    let intrinsic = fx.tcx.item_name(def_id).as_str();
    let intrinsic = &intrinsic[..];

    intrinsic_match! {
        fx, intrinsic, substs, args,
        _ => {
            fx.tcx.sess.span_fatal(span, &format!("Unknown SIMD intrinsic {}", intrinsic));
        };

        simd_cast, (c a) {
            validate_simd_type!(fx, intrinsic, span, a.layout().ty);
            simd_for_each_lane(fx, a, ret, |fx, lane_layout, ret_lane_layout, lane| {
                let ret_lane_ty = fx.clif_type(ret_lane_layout.ty).unwrap();

                let from_signed = type_sign(lane_layout.ty);
                let to_signed = type_sign(ret_lane_layout.ty);

                let ret_lane = clif_int_or_float_cast(fx, lane, from_signed, ret_lane_ty, to_signed);
                CValue::by_val(ret_lane, ret_lane_layout)
            });
        };

        simd_eq, (c x, c y) {
            validate_simd_type!(fx, intrinsic, span, x.layout().ty);
            simd_cmp!(fx, Equal|Equal(x, y) -> ret);
        };
        simd_ne, (c x, c y) {
            validate_simd_type!(fx, intrinsic, span, x.layout().ty);
            simd_cmp!(fx, NotEqual|NotEqual(x, y) -> ret);
        };
        simd_lt, (c x, c y) {
            validate_simd_type!(fx, intrinsic, span, x.layout().ty);
            simd_cmp!(fx, UnsignedLessThan|SignedLessThan|LessThan(x, y) -> ret);
        };
        simd_le, (c x, c y) {
            validate_simd_type!(fx, intrinsic, span, x.layout().ty);
            simd_cmp!(fx, UnsignedLessThanOrEqual|SignedLessThanOrEqual|LessThanOrEqual(x, y) -> ret);
        };
        simd_gt, (c x, c y) {
            validate_simd_type!(fx, intrinsic, span, x.layout().ty);
            simd_cmp!(fx, UnsignedGreaterThan|SignedGreaterThan|GreaterThan(x, y) -> ret);
        };
        simd_ge, (c x, c y) {
            validate_simd_type!(fx, intrinsic, span, x.layout().ty);
            simd_cmp!(
                fx,
                UnsignedGreaterThanOrEqual|SignedGreaterThanOrEqual|GreaterThanOrEqual
                (x, y) -> ret
            );
        };

        // simd_shuffle32<T, U>(x: T, y: T, idx: [u32; 32]) -> U
        _ if intrinsic.starts_with("simd_shuffle"), (c x, c y, o idx) {
            validate_simd_type!(fx, intrinsic, span, x.layout().ty);

            let n: u16 = intrinsic["simd_shuffle".len()..].parse().unwrap();

            assert_eq!(x.layout(), y.layout());
            let layout = x.layout();

            let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
            let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);

            assert_eq!(lane_ty, ret_lane_ty);
            assert_eq!(u64::from(n), ret_lane_count);

            let total_len = lane_count * 2;

            let indexes = {
                use rustc_middle::mir::interpret::*;
                let idx_const = crate::constant::mir_operand_get_const_val(fx, idx).expect("simd_shuffle* idx not const");

                let idx_bytes = match idx_const {
                    ConstValue::ByRef { alloc, offset } => {
                        let ptr = Pointer::new(AllocId(0 /* dummy */), offset);
                        let size = Size::from_bytes(4 * u64::from(ret_lane_count) /* size_of([u32; ret_lane_count]) */);
                        alloc.get_bytes(fx, ptr, size).unwrap()
                    }
                    _ => unreachable!("{:?}", idx_const),
                };

                (0..ret_lane_count).map(|i| {
                    let i = usize::try_from(i).unwrap();
                    let idx = rustc_middle::mir::interpret::read_target_uint(
                        fx.tcx.data_layout.endian,
                        &idx_bytes[4*i.. 4*i + 4],
                    ).expect("read_target_uint");
                    u16::try_from(idx).expect("try_from u32")
                }).collect::<Vec<u16>>()
            };

            for &idx in &indexes {
                assert!(u64::from(idx) < total_len, "idx {} out of range 0..{}", idx, total_len);
            }

            for (out_idx, in_idx) in indexes.into_iter().enumerate() {
                let in_lane = if u64::from(in_idx) < lane_count {
                    x.value_field(fx, mir::Field::new(in_idx.into()))
                } else {
                    y.value_field(fx, mir::Field::new(usize::from(in_idx) - usize::try_from(lane_count).unwrap()))
                };
                let out_lane = ret.place_field(fx, mir::Field::new(out_idx));
                out_lane.write_cvalue(fx, in_lane);
            }
        };

        simd_insert, (c base, o idx, c val) {
            // FIXME validate
            let idx_const = if let Some(idx_const) = crate::constant::mir_operand_get_const_val(fx, idx) {
                idx_const
            } else {
                fx.tcx.sess.span_fatal(
                    span,
                    "Index argument for `simd_insert` is not a constant",
                );
            };

            let idx = idx_const.try_to_bits(Size::from_bytes(4 /* u32*/)).unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const));
            let (lane_count, _lane_ty) = base.layout().ty.simd_size_and_type(fx.tcx);
            if idx >= lane_count.into() {
                fx.tcx.sess.span_fatal(fx.mir.span, &format!("[simd_insert] idx {} >= lane_count {}", idx, lane_count));
            }

            ret.write_cvalue(fx, base);
            let ret_lane = ret.place_field(fx, mir::Field::new(idx.try_into().unwrap()));
            ret_lane.write_cvalue(fx, val);
        };

        simd_extract, (c v, o idx) {
            validate_simd_type!(fx, intrinsic, span, v.layout().ty);
            let idx_const = if let Some(idx_const) = crate::constant::mir_operand_get_const_val(fx, idx) {
                idx_const
            } else {
                fx.tcx.sess.span_warn(
                    span,
                    "Index argument for `simd_extract` is not a constant",
                );
                let res = crate::trap::trap_unimplemented_ret_value(
                    fx,
                    ret.layout(),
                    "Index argument for `simd_extract` is not a constant",
                );
                ret.write_cvalue(fx, res);
                return;
            };

            let idx = idx_const.try_to_bits(Size::from_bytes(4 /* u32*/)).unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const));
            let (lane_count, _lane_ty) = v.layout().ty.simd_size_and_type(fx.tcx);
            if idx >= lane_count.into() {
                fx.tcx.sess.span_fatal(fx.mir.span, &format!("[simd_extract] idx {} >= lane_count {}", idx, lane_count));
            }

            let ret_lane = v.value_field(fx, mir::Field::new(idx.try_into().unwrap()));
            ret.write_cvalue(fx, ret_lane);
        };

        simd_add, (c x, c y) {
            validate_simd_type!(fx, intrinsic, span, x.layout().ty);
            simd_int_flt_binop!(fx, iadd|fadd(x, y) -> ret);
        };
        simd_sub, (c x, c y) {
            validate_simd_type!(fx, intrinsic, span, x.layout().ty);
            simd_int_flt_binop!(fx, isub|fsub(x, y) -> ret);
        };
        simd_mul, (c x, c y) {
            validate_simd_type!(fx, intrinsic, span, x.layout().ty);
            simd_int_flt_binop!(fx, imul|fmul(x, y) -> ret);
        };
        simd_div, (c x, c y) {
            validate_simd_type!(fx, intrinsic, span, x.layout().ty);
            simd_int_flt_binop!(fx, udiv|sdiv|fdiv(x, y) -> ret);
        };
        simd_shl, (c x, c y) {
            validate_simd_type!(fx, intrinsic, span, x.layout().ty);
            simd_int_binop!(fx, ishl(x, y) -> ret);
        };
        simd_shr, (c x, c y) {
            validate_simd_type!(fx, intrinsic, span, x.layout().ty);
            simd_int_binop!(fx, ushr|sshr(x, y) -> ret);
        };
        simd_and, (c x, c y) {
            validate_simd_type!(fx, intrinsic, span, x.layout().ty);
            simd_int_binop!(fx, band(x, y) -> ret);
        };
        simd_or, (c x, c y) {
            validate_simd_type!(fx, intrinsic, span, x.layout().ty);
            simd_int_binop!(fx, bor(x, y) -> ret);
        };
        simd_xor, (c x, c y) {
            validate_simd_type!(fx, intrinsic, span, x.layout().ty);
            simd_int_binop!(fx, bxor(x, y) -> ret);
        };

        simd_fma, (c a, c b, c c) {
            validate_simd_type!(fx, intrinsic, span, a.layout().ty);
            assert_eq!(a.layout(), b.layout());
            assert_eq!(a.layout(), c.layout());
            let layout = a.layout();

            let (lane_count, _lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
            let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
            assert_eq!(lane_count, ret_lane_count);
            let ret_lane_layout = fx.layout_of(ret_lane_ty);

            for lane in 0..lane_count {
                let lane = mir::Field::new(lane.try_into().unwrap());
                let a_lane = a.value_field(fx, lane).load_scalar(fx);
                let b_lane = b.value_field(fx, lane).load_scalar(fx);
                let c_lane = c.value_field(fx, lane).load_scalar(fx);

                let mul_lane = fx.bcx.ins().fmul(a_lane, b_lane);
                let res_lane = CValue::by_val(fx.bcx.ins().fadd(mul_lane, c_lane), ret_lane_layout);

                ret.place_field(fx, lane).write_cvalue(fx, res_lane);
            }
        };

        simd_fmin, (c x, c y) {
            validate_simd_type!(fx, intrinsic, span, x.layout().ty);
            simd_flt_binop!(fx, fmin(x, y) -> ret);
        };
        simd_fmax, (c x, c y) {
            validate_simd_type!(fx, intrinsic, span, x.layout().ty);
            simd_flt_binop!(fx, fmax(x, y) -> ret);
        };

        simd_reduce_add_ordered | simd_reduce_add_unordered, (c v) {
            validate_simd_type!(fx, intrinsic, span, v.layout().ty);
            simd_reduce(fx, v, ret, |fx, lane_layout, a, b| {
                if lane_layout.ty.is_floating_point() {
                    fx.bcx.ins().fadd(a, b)
                } else {
                    fx.bcx.ins().iadd(a, b)
                }
            });
        };

        simd_reduce_mul_ordered | simd_reduce_mul_unordered, (c v) {
            validate_simd_type!(fx, intrinsic, span, v.layout().ty);
            simd_reduce(fx, v, ret, |fx, lane_layout, a, b| {
                if lane_layout.ty.is_floating_point() {
                    fx.bcx.ins().fmul(a, b)
                } else {
                    fx.bcx.ins().imul(a, b)
                }
            });
        };

        simd_reduce_all, (c v) {
            validate_simd_type!(fx, intrinsic, span, v.layout().ty);
            simd_reduce_bool(fx, v, ret, |fx, a, b| fx.bcx.ins().band(a, b));
        };

        simd_reduce_any, (c v) {
            validate_simd_type!(fx, intrinsic, span, v.layout().ty);
            simd_reduce_bool(fx, v, ret, |fx, a, b| fx.bcx.ins().bor(a, b));
        };

        // simd_fabs
        // simd_saturating_add
        // simd_bitmask
        // simd_select
        // simd_rem
        // simd_neg
    }
}
Commit	Line	Data
29967ef6 XL	1	//! Codegen `extern "platform-intrinsic"` intrinsics.
	2
	3	use super::*;
	4	use crate::prelude::*;
	5
	6	pub(super) fn codegen_simd_intrinsic_call<'tcx>(
6a06907d	7	fx: &mut FunctionCx<'_, '_, 'tcx>,
29967ef6 XL	8	instance: Instance<'tcx>,
	9	args: &[mir::Operand<'tcx>],
	10	ret: CPlace<'tcx>,
	11	span: Span,
	12	) {
	13	let def_id = instance.def_id();
	14	let substs = instance.substs;
	15
	16	let intrinsic = fx.tcx.item_name(def_id).as_str();
	17	let intrinsic = &intrinsic[..];
	18
	19	intrinsic_match! {
	20	fx, intrinsic, substs, args,
	21	_ => {
	22	fx.tcx.sess.span_fatal(span, &format!("Unknown SIMD intrinsic {}", intrinsic));
	23	};
	24
	25	simd_cast, (c a) {
	26	validate_simd_type!(fx, intrinsic, span, a.layout().ty);
	27	simd_for_each_lane(fx, a, ret, \|fx, lane_layout, ret_lane_layout, lane\| {
	28	let ret_lane_ty = fx.clif_type(ret_lane_layout.ty).unwrap();
	29
	30	let from_signed = type_sign(lane_layout.ty);
	31	let to_signed = type_sign(ret_lane_layout.ty);
	32
	33	let ret_lane = clif_int_or_float_cast(fx, lane, from_signed, ret_lane_ty, to_signed);
	34	CValue::by_val(ret_lane, ret_lane_layout)
	35	});
	36	};
	37
29967ef6 XL	38	simd_eq, (c x, c y) {
29967ef6 XL	39	validate_simd_type!(fx, intrinsic, span, x.layout().ty);
fc512014	40	simd_cmp!(fx, Equal\|Equal(x, y) -> ret);
29967ef6 XL	41	};
	42	simd_ne, (c x, c y) {
	43	validate_simd_type!(fx, intrinsic, span, x.layout().ty);
fc512014	44	simd_cmp!(fx, NotEqual\|NotEqual(x, y) -> ret);
29967ef6 XL	45	};
	46	simd_lt, (c x, c y) {
	47	validate_simd_type!(fx, intrinsic, span, x.layout().ty);
fc512014	48	simd_cmp!(fx, UnsignedLessThan\|SignedLessThan\|LessThan(x, y) -> ret);
29967ef6 XL	49	};
	50	simd_le, (c x, c y) {
	51	validate_simd_type!(fx, intrinsic, span, x.layout().ty);
fc512014	52	simd_cmp!(fx, UnsignedLessThanOrEqual\|SignedLessThanOrEqual\|LessThanOrEqual(x, y) -> ret);
29967ef6 XL	53	};
	54	simd_gt, (c x, c y) {
	55	validate_simd_type!(fx, intrinsic, span, x.layout().ty);
fc512014	56	simd_cmp!(fx, UnsignedGreaterThan\|SignedGreaterThan\|GreaterThan(x, y) -> ret);
29967ef6 XL	57	};
	58	simd_ge, (c x, c y) {
	59	validate_simd_type!(fx, intrinsic, span, x.layout().ty);
fc512014 XL	60	simd_cmp!(
	61	fx,
	62	UnsignedGreaterThanOrEqual\|SignedGreaterThanOrEqual\|GreaterThanOrEqual
	63	(x, y) -> ret
	64	);
29967ef6 XL	65	};
	66
	67	// simd_shuffle32<T, U>(x: T, y: T, idx: [u32; 32]) -> U
	68	_ if intrinsic.starts_with("simd_shuffle"), (c x, c y, o idx) {
	69	validate_simd_type!(fx, intrinsic, span, x.layout().ty);
	70
	71	let n: u16 = intrinsic["simd_shuffle".len()..].parse().unwrap();
	72
	73	assert_eq!(x.layout(), y.layout());
	74	let layout = x.layout();
	75
5869c6ff XL	76	let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
5869c6ff XL	77	let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
29967ef6	78
5869c6ff XL	79	assert_eq!(lane_ty, ret_lane_ty);
5869c6ff XL	80	assert_eq!(u64::from(n), ret_lane_count);
29967ef6 XL	81
	82	let total_len = lane_count * 2;
	83
	84	let indexes = {
	85	use rustc_middle::mir::interpret::*;
	86	let idx_const = crate::constant::mir_operand_get_const_val(fx, idx).expect("simd_shuffle* idx not const");
	87
6a06907d XL	88	let idx_bytes = match idx_const {
6a06907d XL	89	ConstValue::ByRef { alloc, offset } => {
29967ef6 XL	90	let ptr = Pointer::new(AllocId(0 /* dummy */), offset);
	91	let size = Size::from_bytes(4 * u64::from(ret_lane_count) /* size_of([u32; ret_lane_count]) */);
	92	alloc.get_bytes(fx, ptr, size).unwrap()
	93	}
	94	_ => unreachable!("{:?}", idx_const),
	95	};
	96
	97	(0..ret_lane_count).map(\|i\| {
	98	let i = usize::try_from(i).unwrap();
	99	let idx = rustc_middle::mir::interpret::read_target_uint(
	100	fx.tcx.data_layout.endian,
	101	&idx_bytes[4i.. 4i + 4],
	102	).expect("read_target_uint");
	103	u16::try_from(idx).expect("try_from u32")
	104	}).collect::<Vec<u16>>()
	105	};
	106
	107	for &idx in &indexes {
5869c6ff	108	assert!(u64::from(idx) < total_len, "idx {} out of range 0..{}", idx, total_len);
29967ef6 XL	109	}
	110
	111	for (out_idx, in_idx) in indexes.into_iter().enumerate() {
5869c6ff	112	let in_lane = if u64::from(in_idx) < lane_count {
fc512014	113	x.value_field(fx, mir::Field::new(in_idx.into()))
29967ef6	114	} else {
5869c6ff	115	y.value_field(fx, mir::Field::new(usize::from(in_idx) - usize::try_from(lane_count).unwrap()))
29967ef6 XL	116	};
	117	let out_lane = ret.place_field(fx, mir::Field::new(out_idx));
	118	out_lane.write_cvalue(fx, in_lane);
	119	}
	120	};
	121
	122	simd_insert, (c base, o idx, c val) {
	123	// FIXME validate
	124	let idx_const = if let Some(idx_const) = crate::constant::mir_operand_get_const_val(fx, idx) {
	125	idx_const
	126	} else {
	127	fx.tcx.sess.span_fatal(
	128	span,
	129	"Index argument for `simd_insert` is not a constant",
	130	);
	131	};
	132
6a06907d	133	let idx = idx_const.try_to_bits(Size::from_bytes(4 /* u32*/)).unwrap_or_else(\|\| panic!("kind not scalar: {:?}", idx_const));
5869c6ff	134	let (lane_count, _lane_ty) = base.layout().ty.simd_size_and_type(fx.tcx);
29967ef6 XL	135	if idx >= lane_count.into() {
	136	fx.tcx.sess.span_fatal(fx.mir.span, &format!("[simd_insert] idx {} >= lane_count {}", idx, lane_count));
	137	}
	138
	139	ret.write_cvalue(fx, base);
	140	let ret_lane = ret.place_field(fx, mir::Field::new(idx.try_into().unwrap()));
	141	ret_lane.write_cvalue(fx, val);
	142	};
	143
	144	simd_extract, (c v, o idx) {
	145	validate_simd_type!(fx, intrinsic, span, v.layout().ty);
	146	let idx_const = if let Some(idx_const) = crate::constant::mir_operand_get_const_val(fx, idx) {
	147	idx_const
	148	} else {
fc512014	149	fx.tcx.sess.span_warn(
29967ef6 XL	150	span,
	151	"Index argument for `simd_extract` is not a constant",
	152	);
fc512014 XL	153	let res = crate::trap::trap_unimplemented_ret_value(
	154	fx,
	155	ret.layout(),
	156	"Index argument for `simd_extract` is not a constant",
	157	);
	158	ret.write_cvalue(fx, res);
	159	return;
29967ef6 XL	160	};
29967ef6 XL	161
6a06907d	162	let idx = idx_const.try_to_bits(Size::from_bytes(4 /* u32*/)).unwrap_or_else(\|\| panic!("kind not scalar: {:?}", idx_const));
5869c6ff	163	let (lane_count, _lane_ty) = v.layout().ty.simd_size_and_type(fx.tcx);
29967ef6 XL	164	if idx >= lane_count.into() {
	165	fx.tcx.sess.span_fatal(fx.mir.span, &format!("[simd_extract] idx {} >= lane_count {}", idx, lane_count));
	166	}
	167
	168	let ret_lane = v.value_field(fx, mir::Field::new(idx.try_into().unwrap()));
	169	ret.write_cvalue(fx, ret_lane);
	170	};
	171
	172	simd_add, (c x, c y) {
	173	validate_simd_type!(fx, intrinsic, span, x.layout().ty);
	174	simd_int_flt_binop!(fx, iadd\|fadd(x, y) -> ret);
	175	};
	176	simd_sub, (c x, c y) {
	177	validate_simd_type!(fx, intrinsic, span, x.layout().ty);
	178	simd_int_flt_binop!(fx, isub\|fsub(x, y) -> ret);
	179	};
	180	simd_mul, (c x, c y) {
	181	validate_simd_type!(fx, intrinsic, span, x.layout().ty);
	182	simd_int_flt_binop!(fx, imul\|fmul(x, y) -> ret);
	183	};
	184	simd_div, (c x, c y) {
	185	validate_simd_type!(fx, intrinsic, span, x.layout().ty);
	186	simd_int_flt_binop!(fx, udiv\|sdiv\|fdiv(x, y) -> ret);
	187	};
	188	simd_shl, (c x, c y) {
	189	validate_simd_type!(fx, intrinsic, span, x.layout().ty);
	190	simd_int_binop!(fx, ishl(x, y) -> ret);
	191	};
	192	simd_shr, (c x, c y) {
	193	validate_simd_type!(fx, intrinsic, span, x.layout().ty);
	194	simd_int_binop!(fx, ushr\|sshr(x, y) -> ret);
	195	};
	196	simd_and, (c x, c y) {
	197	validate_simd_type!(fx, intrinsic, span, x.layout().ty);
	198	simd_int_binop!(fx, band(x, y) -> ret);
	199	};
	200	simd_or, (c x, c y) {
	201	validate_simd_type!(fx, intrinsic, span, x.layout().ty);
	202	simd_int_binop!(fx, bor(x, y) -> ret);
	203	};
	204	simd_xor, (c x, c y) {
	205	validate_simd_type!(fx, intrinsic, span, x.layout().ty);
	206	simd_int_binop!(fx, bxor(x, y) -> ret);
	207	};
	208
	209	simd_fma, (c a, c b, c c) {
	210	validate_simd_type!(fx, intrinsic, span, a.layout().ty);
	211	assert_eq!(a.layout(), b.layout());
	212	assert_eq!(a.layout(), c.layout());
	213	let layout = a.layout();
	214
5869c6ff XL	215	let (lane_count, _lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
5869c6ff XL	216	let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
29967ef6	217	assert_eq!(lane_count, ret_lane_count);
5869c6ff	218	let ret_lane_layout = fx.layout_of(ret_lane_ty);
29967ef6 XL	219
29967ef6 XL	220	for lane in 0..lane_count {
5869c6ff	221	let lane = mir::Field::new(lane.try_into().unwrap());
29967ef6 XL	222	let a_lane = a.value_field(fx, lane).load_scalar(fx);
	223	let b_lane = b.value_field(fx, lane).load_scalar(fx);
	224	let c_lane = c.value_field(fx, lane).load_scalar(fx);
	225
	226	let mul_lane = fx.bcx.ins().fmul(a_lane, b_lane);
	227	let res_lane = CValue::by_val(fx.bcx.ins().fadd(mul_lane, c_lane), ret_lane_layout);
	228
	229	ret.place_field(fx, lane).write_cvalue(fx, res_lane);
	230	}
	231	};
	232
	233	simd_fmin, (c x, c y) {
	234	validate_simd_type!(fx, intrinsic, span, x.layout().ty);
	235	simd_flt_binop!(fx, fmin(x, y) -> ret);
	236	};
	237	simd_fmax, (c x, c y) {
	238	validate_simd_type!(fx, intrinsic, span, x.layout().ty);
	239	simd_flt_binop!(fx, fmax(x, y) -> ret);
	240	};
	241
fc512014 XL	242	simd_reduce_add_ordered \| simd_reduce_add_unordered, (c v) {
	243	validate_simd_type!(fx, intrinsic, span, v.layout().ty);
	244	simd_reduce(fx, v, ret, \|fx, lane_layout, a, b\| {
	245	if lane_layout.ty.is_floating_point() {
	246	fx.bcx.ins().fadd(a, b)
	247	} else {
	248	fx.bcx.ins().iadd(a, b)
	249	}
	250	});
	251	};
	252
	253	simd_reduce_mul_ordered \| simd_reduce_mul_unordered, (c v) {
	254	validate_simd_type!(fx, intrinsic, span, v.layout().ty);
	255	simd_reduce(fx, v, ret, \|fx, lane_layout, a, b\| {
	256	if lane_layout.ty.is_floating_point() {
	257	fx.bcx.ins().fmul(a, b)
	258	} else {
	259	fx.bcx.ins().imul(a, b)
	260	}
	261	});
	262	};
	263
	264	simd_reduce_all, (c v) {
	265	validate_simd_type!(fx, intrinsic, span, v.layout().ty);
	266	simd_reduce_bool(fx, v, ret, \|fx, a, b\| fx.bcx.ins().band(a, b));
	267	};
	268
	269	simd_reduce_any, (c v) {
	270	validate_simd_type!(fx, intrinsic, span, v.layout().ty);
	271	simd_reduce_bool(fx, v, ret, \|fx, a, b\| fx.bcx.ins().bor(a, b));
	272	};
	273
29967ef6 XL	274	// simd_fabs
	275	// simd_saturating_add
	276	// simd_bitmask
	277	// simd_select
29967ef6	278	// simd_rem
6a06907d	279	// simd_neg
29967ef6 XL	280	}
29967ef6 XL	281	}