[rustc.git] / compiler / rustc_codegen_cranelift / src / intrinsics / simd.rs

//! Codegen `extern "platform-intrinsic"` intrinsics.

use rustc_middle::ty::subst::SubstsRef;
use rustc_span::Symbol;

use super::*;
use crate::prelude::*;

fn report_simd_type_validation_error(
    fx: &mut FunctionCx<'_, '_, '_>,
    intrinsic: Symbol,
    span: Span,
    ty: Ty<'_>,
) {
    fx.tcx.sess.span_err(span, &format!("invalid monomorphization of `{}` intrinsic: expected SIMD input type, found non-SIMD `{}`", intrinsic, ty));
    // Prevent verifier error
    crate::trap::trap_unreachable(fx, "compilation should not have succeeded");
}

pub(super) fn codegen_simd_intrinsic_call<'tcx>(
    fx: &mut FunctionCx<'_, '_, 'tcx>,
    intrinsic: Symbol,
    _substs: SubstsRef<'tcx>,
    args: &[mir::Operand<'tcx>],
    ret: CPlace<'tcx>,
    span: Span,
) {
    intrinsic_match! {
        fx, intrinsic, args,
        _ => {
            fx.tcx.sess.span_fatal(span, &format!("Unknown SIMD intrinsic {}", intrinsic));
        };

        simd_cast, (c a) {
            if !a.layout().ty.is_simd() {
                report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
                return;
            }

            simd_for_each_lane(fx, a, ret, &|fx, lane_ty, ret_lane_ty, lane| {
                let ret_lane_clif_ty = fx.clif_type(ret_lane_ty).unwrap();

                let from_signed = type_sign(lane_ty);
                let to_signed = type_sign(ret_lane_ty);

                clif_int_or_float_cast(fx, lane, from_signed, ret_lane_clif_ty, to_signed)
            });
        };

        simd_eq | simd_ne | simd_lt | simd_le | simd_gt | simd_ge, (c x, c y) {
            if !x.layout().ty.is_simd() {
                report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty);
                return;
            }

            // FIXME use vector instructions when possible
            simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, res_lane_ty, x_lane, y_lane| {
                let res_lane = match (lane_ty.kind(), intrinsic) {
                    (ty::Uint(_), sym::simd_eq) => fx.bcx.ins().icmp(IntCC::Equal, x_lane, y_lane),
                    (ty::Uint(_), sym::simd_ne) => fx.bcx.ins().icmp(IntCC::NotEqual, x_lane, y_lane),
                    (ty::Uint(_), sym::simd_lt) => {
                        fx.bcx.ins().icmp(IntCC::UnsignedLessThan, x_lane, y_lane)
                    }
                    (ty::Uint(_), sym::simd_le) => {
                        fx.bcx.ins().icmp(IntCC::UnsignedLessThanOrEqual, x_lane, y_lane)
                    }
                    (ty::Uint(_), sym::simd_gt) => {
                        fx.bcx.ins().icmp(IntCC::UnsignedGreaterThan, x_lane, y_lane)
                    }
                    (ty::Uint(_), sym::simd_ge) => {
                        fx.bcx.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, x_lane, y_lane)
                    }

                    (ty::Int(_), sym::simd_eq) => fx.bcx.ins().icmp(IntCC::Equal, x_lane, y_lane),
                    (ty::Int(_), sym::simd_ne) => fx.bcx.ins().icmp(IntCC::NotEqual, x_lane, y_lane),
                    (ty::Int(_), sym::simd_lt) => fx.bcx.ins().icmp(IntCC::SignedLessThan, x_lane, y_lane),
                    (ty::Int(_), sym::simd_le) => {
                        fx.bcx.ins().icmp(IntCC::SignedLessThanOrEqual, x_lane, y_lane)
                    }
                    (ty::Int(_), sym::simd_gt) => {
                        fx.bcx.ins().icmp(IntCC::SignedGreaterThan, x_lane, y_lane)
                    }
                    (ty::Int(_), sym::simd_ge) => {
                        fx.bcx.ins().icmp(IntCC::SignedGreaterThanOrEqual, x_lane, y_lane)
                    }

                    (ty::Float(_), sym::simd_eq) => fx.bcx.ins().fcmp(FloatCC::Equal, x_lane, y_lane),
                    (ty::Float(_), sym::simd_ne) => fx.bcx.ins().fcmp(FloatCC::NotEqual, x_lane, y_lane),
                    (ty::Float(_), sym::simd_lt) => fx.bcx.ins().fcmp(FloatCC::LessThan, x_lane, y_lane),
                    (ty::Float(_), sym::simd_le) => {
                        fx.bcx.ins().fcmp(FloatCC::LessThanOrEqual, x_lane, y_lane)
                    }
                    (ty::Float(_), sym::simd_gt) => fx.bcx.ins().fcmp(FloatCC::GreaterThan, x_lane, y_lane),
                    (ty::Float(_), sym::simd_ge) => {
                        fx.bcx.ins().fcmp(FloatCC::GreaterThanOrEqual, x_lane, y_lane)
                    }

                    _ => unreachable!(),
                };

                let ty = fx.clif_type(res_lane_ty).unwrap();

                let res_lane = fx.bcx.ins().bint(ty, res_lane);
                fx.bcx.ins().ineg(res_lane)
            });
        };

        // simd_shuffle32<T, U>(x: T, y: T, idx: [u32; 32]) -> U
        _ if intrinsic.as_str().starts_with("simd_shuffle"), (c x, c y, o idx) {
            if !x.layout().ty.is_simd() {
                report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty);
                return;
            }

            // If this intrinsic is the older "simd_shuffleN" form, simply parse the integer.
            // If there is no suffix, use the index array length.
            let n: u16 = if intrinsic == sym::simd_shuffle {
                // Make sure this is actually an array, since typeck only checks the length-suffixed
                // version of this intrinsic.
                let idx_ty = fx.monomorphize(idx.ty(fx.mir, fx.tcx));
                match idx_ty.kind() {
                    ty::Array(ty, len) if matches!(ty.kind(), ty::Uint(ty::UintTy::U32)) => {
                        len.try_eval_usize(fx.tcx, ty::ParamEnv::reveal_all()).unwrap_or_else(|| {
                            span_bug!(span, "could not evaluate shuffle index array length")
                        }).try_into().unwrap()
                    }
                    _ => {
                        fx.tcx.sess.span_err(
                            span,
                            &format!(
                                "simd_shuffle index must be an array of `u32`, got `{}`",
                                idx_ty,
                            ),
                        );
                        // Prevent verifier error
                        crate::trap::trap_unreachable(fx, "compilation should not have succeeded");
                        return;
                    }
                }
            } else {
                intrinsic.as_str()["simd_shuffle".len()..].parse().unwrap()
            };

            assert_eq!(x.layout(), y.layout());
            let layout = x.layout();

            let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
            let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);

            assert_eq!(lane_ty, ret_lane_ty);
            assert_eq!(u64::from(n), ret_lane_count);

            let total_len = lane_count * 2;

            let indexes = {
                use rustc_middle::mir::interpret::*;
                let idx_const = crate::constant::mir_operand_get_const_val(fx, idx).expect("simd_shuffle* idx not const");

                let idx_bytes = match idx_const {
                    ConstValue::ByRef { alloc, offset } => {
                        let size = Size::from_bytes(4 * ret_lane_count /* size_of([u32; ret_lane_count]) */);
                        alloc.inner().get_bytes(fx, alloc_range(offset, size)).unwrap()
                    }
                    _ => unreachable!("{:?}", idx_const),
                };

                (0..ret_lane_count).map(|i| {
                    let i = usize::try_from(i).unwrap();
                    let idx = rustc_middle::mir::interpret::read_target_uint(
                        fx.tcx.data_layout.endian,
                        &idx_bytes[4*i.. 4*i + 4],
                    ).expect("read_target_uint");
                    u16::try_from(idx).expect("try_from u32")
                }).collect::<Vec<u16>>()
            };

            for &idx in &indexes {
                assert!(u64::from(idx) < total_len, "idx {} out of range 0..{}", idx, total_len);
            }

            for (out_idx, in_idx) in indexes.into_iter().enumerate() {
                let in_lane = if u64::from(in_idx) < lane_count {
                    x.value_lane(fx, in_idx.into())
                } else {
                    y.value_lane(fx, u64::from(in_idx) - lane_count)
                };
                let out_lane = ret.place_lane(fx, u64::try_from(out_idx).unwrap());
                out_lane.write_cvalue(fx, in_lane);
            }
        };

        simd_insert, (c base, o idx, c val) {
            // FIXME validate
            let idx_const = if let Some(idx_const) = crate::constant::mir_operand_get_const_val(fx, idx) {
                idx_const
            } else {
                fx.tcx.sess.span_fatal(
                    span,
                    "Index argument for `simd_insert` is not a constant",
                );
            };

            let idx = idx_const.try_to_bits(Size::from_bytes(4 /* u32*/)).unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const));
            let (lane_count, _lane_ty) = base.layout().ty.simd_size_and_type(fx.tcx);
            if idx >= lane_count.into() {
                fx.tcx.sess.span_fatal(fx.mir.span, &format!("[simd_insert] idx {} >= lane_count {}", idx, lane_count));
            }

            ret.write_cvalue(fx, base);
            let ret_lane = ret.place_field(fx, mir::Field::new(idx.try_into().unwrap()));
            ret_lane.write_cvalue(fx, val);
        };

        simd_extract, (c v, o idx) {
            if !v.layout().ty.is_simd() {
                report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
                return;
            }

            let idx_const = if let Some(idx_const) = crate::constant::mir_operand_get_const_val(fx, idx) {
                idx_const
            } else {
                fx.tcx.sess.span_warn(
                    span,
                    "Index argument for `simd_extract` is not a constant",
                );
                let res = crate::trap::trap_unimplemented_ret_value(
                    fx,
                    ret.layout(),
                    "Index argument for `simd_extract` is not a constant",
                );
                ret.write_cvalue(fx, res);
                return;
            };

            let idx = idx_const.try_to_bits(Size::from_bytes(4 /* u32*/)).unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const));
            let (lane_count, _lane_ty) = v.layout().ty.simd_size_and_type(fx.tcx);
            if idx >= lane_count.into() {
                fx.tcx.sess.span_fatal(fx.mir.span, &format!("[simd_extract] idx {} >= lane_count {}", idx, lane_count));
            }

            let ret_lane = v.value_lane(fx, idx.try_into().unwrap());
            ret.write_cvalue(fx, ret_lane);
        };

        simd_neg, (c a) {
            if !a.layout().ty.is_simd() {
                report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
                return;
            }

            simd_for_each_lane(fx, a, ret, &|fx, lane_ty, _ret_lane_ty, lane| {
                match lane_ty.kind() {
                    ty::Int(_) => fx.bcx.ins().ineg(lane),
                    ty::Float(_) => fx.bcx.ins().fneg(lane),
                    _ => unreachable!(),
                }
            });
        };

        simd_add | simd_sub | simd_mul | simd_div | simd_rem
        | simd_shl | simd_shr | simd_and | simd_or | simd_xor, (c x, c y) {
            if !x.layout().ty.is_simd() {
                report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty);
                return;
            }

            // FIXME use vector instructions when possible
            simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, _ret_lane_ty, x_lane, y_lane| match (
                lane_ty.kind(),
                intrinsic,
            ) {
                (ty::Uint(_), sym::simd_add) => fx.bcx.ins().iadd(x_lane, y_lane),
                (ty::Uint(_), sym::simd_sub) => fx.bcx.ins().isub(x_lane, y_lane),
                (ty::Uint(_), sym::simd_mul) => fx.bcx.ins().imul(x_lane, y_lane),
                (ty::Uint(_), sym::simd_div) => fx.bcx.ins().udiv(x_lane, y_lane),
                (ty::Uint(_), sym::simd_rem) => fx.bcx.ins().urem(x_lane, y_lane),

                (ty::Int(_), sym::simd_add) => fx.bcx.ins().iadd(x_lane, y_lane),
                (ty::Int(_), sym::simd_sub) => fx.bcx.ins().isub(x_lane, y_lane),
                (ty::Int(_), sym::simd_mul) => fx.bcx.ins().imul(x_lane, y_lane),
                (ty::Int(_), sym::simd_div) => fx.bcx.ins().sdiv(x_lane, y_lane),
                (ty::Int(_), sym::simd_rem) => fx.bcx.ins().srem(x_lane, y_lane),

                (ty::Float(_), sym::simd_add) => fx.bcx.ins().fadd(x_lane, y_lane),
                (ty::Float(_), sym::simd_sub) => fx.bcx.ins().fsub(x_lane, y_lane),
                (ty::Float(_), sym::simd_mul) => fx.bcx.ins().fmul(x_lane, y_lane),
                (ty::Float(_), sym::simd_div) => fx.bcx.ins().fdiv(x_lane, y_lane),
                (ty::Float(FloatTy::F32), sym::simd_rem) => fx.lib_call(
                    "fmodf",
                    vec![AbiParam::new(types::F32), AbiParam::new(types::F32)],
                    vec![AbiParam::new(types::F32)],
                    &[x_lane, y_lane],
                )[0],
                (ty::Float(FloatTy::F64), sym::simd_rem) => fx.lib_call(
                    "fmod",
                    vec![AbiParam::new(types::F64), AbiParam::new(types::F64)],
                    vec![AbiParam::new(types::F64)],
                    &[x_lane, y_lane],
                )[0],

                (ty::Uint(_), sym::simd_shl) => fx.bcx.ins().ishl(x_lane, y_lane),
                (ty::Uint(_), sym::simd_shr) => fx.bcx.ins().ushr(x_lane, y_lane),
                (ty::Uint(_), sym::simd_and) => fx.bcx.ins().band(x_lane, y_lane),
                (ty::Uint(_), sym::simd_or) => fx.bcx.ins().bor(x_lane, y_lane),
                (ty::Uint(_), sym::simd_xor) => fx.bcx.ins().bxor(x_lane, y_lane),

                (ty::Int(_), sym::simd_shl) => fx.bcx.ins().ishl(x_lane, y_lane),
                (ty::Int(_), sym::simd_shr) => fx.bcx.ins().sshr(x_lane, y_lane),
                (ty::Int(_), sym::simd_and) => fx.bcx.ins().band(x_lane, y_lane),
                (ty::Int(_), sym::simd_or) => fx.bcx.ins().bor(x_lane, y_lane),
                (ty::Int(_), sym::simd_xor) => fx.bcx.ins().bxor(x_lane, y_lane),

                _ => unreachable!(),
            });
        };

        simd_fma, (c a, c b, c c) {
            if !a.layout().ty.is_simd() {
                report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
                return;
            }
            assert_eq!(a.layout(), b.layout());
            assert_eq!(a.layout(), c.layout());
            let layout = a.layout();

            let (lane_count, _lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
            let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
            assert_eq!(lane_count, ret_lane_count);
            let ret_lane_layout = fx.layout_of(ret_lane_ty);

            for lane in 0..lane_count {
                let a_lane = a.value_lane(fx, lane).load_scalar(fx);
                let b_lane = b.value_lane(fx, lane).load_scalar(fx);
                let c_lane = c.value_lane(fx, lane).load_scalar(fx);

                let mul_lane = fx.bcx.ins().fmul(a_lane, b_lane);
                let res_lane = CValue::by_val(fx.bcx.ins().fadd(mul_lane, c_lane), ret_lane_layout);

                ret.place_lane(fx, lane).write_cvalue(fx, res_lane);
            }
        };

        simd_fmin | simd_fmax, (c x, c y) {
            if !x.layout().ty.is_simd() {
                report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty);
                return;
            }

            // FIXME use vector instructions when possible
            simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, _ret_lane_ty, x_lane, y_lane| {
                match lane_ty.kind() {
                    ty::Float(_) => {},
                    _ => unreachable!("{:?}", lane_ty),
                }
                match intrinsic {
                    sym::simd_fmin => fx.bcx.ins().fmin(x_lane, y_lane),
                    sym::simd_fmax => fx.bcx.ins().fmax(x_lane, y_lane),
                    _ => unreachable!(),
                }
            });
        };

        simd_round, (c a) {
            if !a.layout().ty.is_simd() {
                report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
                return;
            }

            simd_for_each_lane(fx, a, ret, &|fx, lane_ty, _ret_lane_ty, lane| {
                match lane_ty.kind() {
                    ty::Float(FloatTy::F32) => fx.lib_call(
                        "roundf",
                        vec![AbiParam::new(types::F32)],
                        vec![AbiParam::new(types::F32)],
                        &[lane],
                    )[0],
                    ty::Float(FloatTy::F64) => fx.lib_call(
                        "round",
                        vec![AbiParam::new(types::F64)],
                        vec![AbiParam::new(types::F64)],
                        &[lane],
                    )[0],
                    _ => unreachable!("{:?}", lane_ty),
                }
            });
        };

        simd_fabs | simd_fsqrt | simd_ceil | simd_floor | simd_trunc, (c a) {
            if !a.layout().ty.is_simd() {
                report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
                return;
            }

            simd_for_each_lane(fx, a, ret, &|fx, lane_ty, _ret_lane_ty, lane| {
                match lane_ty.kind() {
                    ty::Float(_) => {},
                    _ => unreachable!("{:?}", lane_ty),
                }
                match intrinsic {
                    sym::simd_fabs => fx.bcx.ins().fabs(lane),
                    sym::simd_fsqrt => fx.bcx.ins().sqrt(lane),
                    sym::simd_ceil => fx.bcx.ins().ceil(lane),
                    sym::simd_floor => fx.bcx.ins().floor(lane),
                    sym::simd_trunc => fx.bcx.ins().trunc(lane),
                    _ => unreachable!(),
                }
            });
        };

        simd_reduce_add_ordered | simd_reduce_add_unordered, (c v, v acc) {
            // FIXME there must be no acc param for integer vectors
            if !v.layout().ty.is_simd() {
                report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
                return;
            }

            simd_reduce(fx, v, Some(acc), ret, &|fx, lane_ty, a, b| {
                if lane_ty.is_floating_point() {
                    fx.bcx.ins().fadd(a, b)
                } else {
                    fx.bcx.ins().iadd(a, b)
                }
            });
        };

        simd_reduce_mul_ordered | simd_reduce_mul_unordered, (c v, v acc) {
            // FIXME there must be no acc param for integer vectors
            if !v.layout().ty.is_simd() {
                report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
                return;
            }

            simd_reduce(fx, v, Some(acc), ret, &|fx, lane_ty, a, b| {
                if lane_ty.is_floating_point() {
                    fx.bcx.ins().fmul(a, b)
                } else {
                    fx.bcx.ins().imul(a, b)
                }
            });
        };

        simd_reduce_all, (c v) {
            if !v.layout().ty.is_simd() {
                report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
                return;
            }

            simd_reduce_bool(fx, v, ret, &|fx, a, b| fx.bcx.ins().band(a, b));
        };

        simd_reduce_any, (c v) {
            if !v.layout().ty.is_simd() {
                report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
                return;
            }

            simd_reduce_bool(fx, v, ret, &|fx, a, b| fx.bcx.ins().bor(a, b));
        };

        simd_reduce_and, (c v) {
            if !v.layout().ty.is_simd() {
                report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
                return;
            }

            simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| fx.bcx.ins().band(a, b));
        };

        simd_reduce_or, (c v) {
            if !v.layout().ty.is_simd() {
                report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
                return;
            }

            simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| fx.bcx.ins().bor(a, b));
        };

        simd_reduce_xor, (c v) {
            if !v.layout().ty.is_simd() {
                report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
                return;
            }

            simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| fx.bcx.ins().bxor(a, b));
        };

        simd_reduce_min, (c v) {
            if !v.layout().ty.is_simd() {
                report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
                return;
            }

            simd_reduce(fx, v, None, ret, &|fx, ty, a, b| {
                let lt = match ty.kind() {
                    ty::Int(_) => fx.bcx.ins().icmp(IntCC::SignedLessThan, a, b),
                    ty::Uint(_) => fx.bcx.ins().icmp(IntCC::UnsignedLessThan, a, b),
                    ty::Float(_) => fx.bcx.ins().fcmp(FloatCC::LessThan, a, b),
                    _ => unreachable!(),
                };
                fx.bcx.ins().select(lt, a, b)
            });
        };

        simd_reduce_max, (c v) {
            if !v.layout().ty.is_simd() {
                report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
                return;
            }

            simd_reduce(fx, v, None, ret, &|fx, ty, a, b| {
                let gt = match ty.kind() {
                    ty::Int(_) => fx.bcx.ins().icmp(IntCC::SignedGreaterThan, a, b),
                    ty::Uint(_) => fx.bcx.ins().icmp(IntCC::UnsignedGreaterThan, a, b),
                    ty::Float(_) => fx.bcx.ins().fcmp(FloatCC::GreaterThan, a, b),
                    _ => unreachable!(),
                };
                fx.bcx.ins().select(gt, a, b)
            });
        };

        simd_select, (c m, c a, c b) {
            if !m.layout().ty.is_simd() {
                report_simd_type_validation_error(fx, intrinsic, span, m.layout().ty);
                return;
            }
            if !a.layout().ty.is_simd() {
                report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
                return;
            }
            assert_eq!(a.layout(), b.layout());

            let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx);
            let lane_layout = fx.layout_of(lane_ty);

            for lane in 0..lane_count {
                let m_lane = m.value_lane(fx, lane).load_scalar(fx);
                let a_lane = a.value_lane(fx, lane).load_scalar(fx);
                let b_lane = b.value_lane(fx, lane).load_scalar(fx);

                let m_lane = fx.bcx.ins().icmp_imm(IntCC::Equal, m_lane, 0);
                let res_lane = CValue::by_val(fx.bcx.ins().select(m_lane, b_lane, a_lane), lane_layout);

                ret.place_lane(fx, lane).write_cvalue(fx, res_lane);
            }
        };

        // simd_saturating_*
        // simd_bitmask
        // simd_scatter
        // simd_gather
    }
}
Commit	Line	Data
29967ef6 XL	1	//! Codegen `extern "platform-intrinsic"` intrinsics.
29967ef6 XL	2
5e7ed085 FG	3	use rustc_middle::ty::subst::SubstsRef;
	4	use rustc_span::Symbol;
	5
29967ef6 XL	6	use super::*;
	7	use crate::prelude::*;
	8
5e7ed085 FG	9	fn report_simd_type_validation_error(
	10	fx: &mut FunctionCx<'_, '_, '_>,
	11	intrinsic: Symbol,
	12	span: Span,
	13	ty: Ty<'_>,
	14	) {
	15	fx.tcx.sess.span_err(span, &format!("invalid monomorphization of `{}` intrinsic: expected SIMD input type, found non-SIMD `{}`", intrinsic, ty));
	16	// Prevent verifier error
	17	crate::trap::trap_unreachable(fx, "compilation should not have succeeded");
	18	}
	19
29967ef6	20	pub(super) fn codegen_simd_intrinsic_call<'tcx>(
6a06907d	21	fx: &mut FunctionCx<'_, '_, 'tcx>,
5e7ed085 FG	22	intrinsic: Symbol,
5e7ed085 FG	23	_substs: SubstsRef<'tcx>,
29967ef6 XL	24	args: &[mir::Operand<'tcx>],
	25	ret: CPlace<'tcx>,
	26	span: Span,
	27	) {
29967ef6	28	intrinsic_match! {
5e7ed085	29	fx, intrinsic, args,
29967ef6 XL	30	_ => {
	31	fx.tcx.sess.span_fatal(span, &format!("Unknown SIMD intrinsic {}", intrinsic));
	32	};
	33
	34	simd_cast, (c a) {
5e7ed085 FG	35	if !a.layout().ty.is_simd() {
	36	report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
	37	return;
	38	}
	39
	40	simd_for_each_lane(fx, a, ret, &\|fx, lane_ty, ret_lane_ty, lane\| {
	41	let ret_lane_clif_ty = fx.clif_type(ret_lane_ty).unwrap();
29967ef6	42
5e7ed085 FG	43	let from_signed = type_sign(lane_ty);
5e7ed085 FG	44	let to_signed = type_sign(ret_lane_ty);
29967ef6	45
5e7ed085	46	clif_int_or_float_cast(fx, lane, from_signed, ret_lane_clif_ty, to_signed)
29967ef6 XL	47	});
	48	};
	49
5e7ed085 FG	50	simd_eq \| simd_ne \| simd_lt \| simd_le \| simd_gt \| simd_ge, (c x, c y) {
	51	if !x.layout().ty.is_simd() {
	52	report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty);
	53	return;
	54	}
	55
	56	// FIXME use vector instructions when possible
	57	simd_pair_for_each_lane(fx, x, y, ret, &\|fx, lane_ty, res_lane_ty, x_lane, y_lane\| {
	58	let res_lane = match (lane_ty.kind(), intrinsic) {
	59	(ty::Uint(_), sym::simd_eq) => fx.bcx.ins().icmp(IntCC::Equal, x_lane, y_lane),
	60	(ty::Uint(_), sym::simd_ne) => fx.bcx.ins().icmp(IntCC::NotEqual, x_lane, y_lane),
	61	(ty::Uint(_), sym::simd_lt) => {
	62	fx.bcx.ins().icmp(IntCC::UnsignedLessThan, x_lane, y_lane)
	63	}
	64	(ty::Uint(_), sym::simd_le) => {
	65	fx.bcx.ins().icmp(IntCC::UnsignedLessThanOrEqual, x_lane, y_lane)
	66	}
	67	(ty::Uint(_), sym::simd_gt) => {
	68	fx.bcx.ins().icmp(IntCC::UnsignedGreaterThan, x_lane, y_lane)
	69	}
	70	(ty::Uint(_), sym::simd_ge) => {
	71	fx.bcx.ins().icmp(IntCC::UnsignedGreaterThanOrEqual, x_lane, y_lane)
	72	}
	73
	74	(ty::Int(_), sym::simd_eq) => fx.bcx.ins().icmp(IntCC::Equal, x_lane, y_lane),
	75	(ty::Int(_), sym::simd_ne) => fx.bcx.ins().icmp(IntCC::NotEqual, x_lane, y_lane),
	76	(ty::Int(_), sym::simd_lt) => fx.bcx.ins().icmp(IntCC::SignedLessThan, x_lane, y_lane),
	77	(ty::Int(_), sym::simd_le) => {
	78	fx.bcx.ins().icmp(IntCC::SignedLessThanOrEqual, x_lane, y_lane)
	79	}
	80	(ty::Int(_), sym::simd_gt) => {
	81	fx.bcx.ins().icmp(IntCC::SignedGreaterThan, x_lane, y_lane)
	82	}
	83	(ty::Int(_), sym::simd_ge) => {
	84	fx.bcx.ins().icmp(IntCC::SignedGreaterThanOrEqual, x_lane, y_lane)
	85	}
	86
	87	(ty::Float(_), sym::simd_eq) => fx.bcx.ins().fcmp(FloatCC::Equal, x_lane, y_lane),
	88	(ty::Float(_), sym::simd_ne) => fx.bcx.ins().fcmp(FloatCC::NotEqual, x_lane, y_lane),
	89	(ty::Float(_), sym::simd_lt) => fx.bcx.ins().fcmp(FloatCC::LessThan, x_lane, y_lane),
	90	(ty::Float(_), sym::simd_le) => {
	91	fx.bcx.ins().fcmp(FloatCC::LessThanOrEqual, x_lane, y_lane)
	92	}
	93	(ty::Float(_), sym::simd_gt) => fx.bcx.ins().fcmp(FloatCC::GreaterThan, x_lane, y_lane),
	94	(ty::Float(_), sym::simd_ge) => {
	95	fx.bcx.ins().fcmp(FloatCC::GreaterThanOrEqual, x_lane, y_lane)
	96	}
	97
	98	_ => unreachable!(),
	99	};
	100
	101	let ty = fx.clif_type(res_lane_ty).unwrap();
	102
	103	let res_lane = fx.bcx.ins().bint(ty, res_lane);
	104	fx.bcx.ins().ineg(res_lane)
	105	});
29967ef6 XL	106	};
	107
	108	// simd_shuffle32<T, U>(x: T, y: T, idx: [u32; 32]) -> U
17df50a5	109	_ if intrinsic.as_str().starts_with("simd_shuffle"), (c x, c y, o idx) {
5e7ed085 FG	110	if !x.layout().ty.is_simd() {
	111	report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty);
	112	return;
	113	}
29967ef6	114
a2a8927a XL	115	// If this intrinsic is the older "simd_shuffleN" form, simply parse the integer.
	116	// If there is no suffix, use the index array length.
	117	let n: u16 = if intrinsic == sym::simd_shuffle {
	118	// Make sure this is actually an array, since typeck only checks the length-suffixed
	119	// version of this intrinsic.
	120	let idx_ty = fx.monomorphize(idx.ty(fx.mir, fx.tcx));
	121	match idx_ty.kind() {
	122	ty::Array(ty, len) if matches!(ty.kind(), ty::Uint(ty::UintTy::U32)) => {
	123	len.try_eval_usize(fx.tcx, ty::ParamEnv::reveal_all()).unwrap_or_else(\|\| {
	124	span_bug!(span, "could not evaluate shuffle index array length")
	125	}).try_into().unwrap()
	126	}
	127	_ => {
	128	fx.tcx.sess.span_err(
	129	span,
	130	&format!(
	131	"simd_shuffle index must be an array of `u32`, got `{}`",
	132	idx_ty,
	133	),
	134	);
	135	// Prevent verifier error
	136	crate::trap::trap_unreachable(fx, "compilation should not have succeeded");
	137	return;
	138	}
	139	}
	140	} else {
	141	intrinsic.as_str()["simd_shuffle".len()..].parse().unwrap()
	142	};
29967ef6 XL	143
	144	assert_eq!(x.layout(), y.layout());
	145	let layout = x.layout();
	146
5869c6ff XL	147	let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
5869c6ff XL	148	let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
29967ef6	149
5869c6ff XL	150	assert_eq!(lane_ty, ret_lane_ty);
5869c6ff XL	151	assert_eq!(u64::from(n), ret_lane_count);
29967ef6 XL	152
	153	let total_len = lane_count * 2;
	154
	155	let indexes = {
	156	use rustc_middle::mir::interpret::*;
	157	let idx_const = crate::constant::mir_operand_get_const_val(fx, idx).expect("simd_shuffle* idx not const");
	158
6a06907d XL	159	let idx_bytes = match idx_const {
6a06907d XL	160	ConstValue::ByRef { alloc, offset } => {
cdc7bbd5	161	let size = Size::from_bytes(4 * ret_lane_count /* size_of([u32; ret_lane_count]) */);
5e7ed085	162	alloc.inner().get_bytes(fx, alloc_range(offset, size)).unwrap()
29967ef6 XL	163	}
	164	_ => unreachable!("{:?}", idx_const),
	165	};
	166
	167	(0..ret_lane_count).map(\|i\| {
	168	let i = usize::try_from(i).unwrap();
	169	let idx = rustc_middle::mir::interpret::read_target_uint(
	170	fx.tcx.data_layout.endian,
	171	&idx_bytes[4i.. 4i + 4],
	172	).expect("read_target_uint");
	173	u16::try_from(idx).expect("try_from u32")
	174	}).collect::<Vec<u16>>()
	175	};
	176
	177	for &idx in &indexes {
5869c6ff	178	assert!(u64::from(idx) < total_len, "idx {} out of range 0..{}", idx, total_len);
29967ef6 XL	179	}
	180
	181	for (out_idx, in_idx) in indexes.into_iter().enumerate() {
5869c6ff	182	let in_lane = if u64::from(in_idx) < lane_count {
94222f64	183	x.value_lane(fx, in_idx.into())
29967ef6	184	} else {
94222f64	185	y.value_lane(fx, u64::from(in_idx) - lane_count)
29967ef6	186	};
94222f64	187	let out_lane = ret.place_lane(fx, u64::try_from(out_idx).unwrap());
29967ef6 XL	188	out_lane.write_cvalue(fx, in_lane);
	189	}
	190	};
	191
	192	simd_insert, (c base, o idx, c val) {
	193	// FIXME validate
	194	let idx_const = if let Some(idx_const) = crate::constant::mir_operand_get_const_val(fx, idx) {
	195	idx_const
	196	} else {
	197	fx.tcx.sess.span_fatal(
	198	span,
	199	"Index argument for `simd_insert` is not a constant",
	200	);
	201	};
	202
6a06907d	203	let idx = idx_const.try_to_bits(Size::from_bytes(4 /* u32*/)).unwrap_or_else(\|\| panic!("kind not scalar: {:?}", idx_const));
5869c6ff	204	let (lane_count, _lane_ty) = base.layout().ty.simd_size_and_type(fx.tcx);
29967ef6 XL	205	if idx >= lane_count.into() {
	206	fx.tcx.sess.span_fatal(fx.mir.span, &format!("[simd_insert] idx {} >= lane_count {}", idx, lane_count));
	207	}
	208
	209	ret.write_cvalue(fx, base);
	210	let ret_lane = ret.place_field(fx, mir::Field::new(idx.try_into().unwrap()));
	211	ret_lane.write_cvalue(fx, val);
	212	};
	213
	214	simd_extract, (c v, o idx) {
5e7ed085 FG	215	if !v.layout().ty.is_simd() {
	216	report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
	217	return;
	218	}
	219
29967ef6 XL	220	let idx_const = if let Some(idx_const) = crate::constant::mir_operand_get_const_val(fx, idx) {
	221	idx_const
	222	} else {
fc512014	223	fx.tcx.sess.span_warn(
29967ef6 XL	224	span,
	225	"Index argument for `simd_extract` is not a constant",
	226	);
fc512014 XL	227	let res = crate::trap::trap_unimplemented_ret_value(
	228	fx,
	229	ret.layout(),
	230	"Index argument for `simd_extract` is not a constant",
	231	);
	232	ret.write_cvalue(fx, res);
	233	return;
29967ef6 XL	234	};
29967ef6 XL	235
6a06907d	236	let idx = idx_const.try_to_bits(Size::from_bytes(4 /* u32*/)).unwrap_or_else(\|\| panic!("kind not scalar: {:?}", idx_const));
5869c6ff	237	let (lane_count, _lane_ty) = v.layout().ty.simd_size_and_type(fx.tcx);
29967ef6 XL	238	if idx >= lane_count.into() {
	239	fx.tcx.sess.span_fatal(fx.mir.span, &format!("[simd_extract] idx {} >= lane_count {}", idx, lane_count));
	240	}
	241
94222f64	242	let ret_lane = v.value_lane(fx, idx.try_into().unwrap());
29967ef6 XL	243	ret.write_cvalue(fx, ret_lane);
	244	};
	245
94222f64	246	simd_neg, (c a) {
5e7ed085 FG	247	if !a.layout().ty.is_simd() {
	248	report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
	249	return;
	250	}
	251
	252	simd_for_each_lane(fx, a, ret, &\|fx, lane_ty, _ret_lane_ty, lane\| {
	253	match lane_ty.kind() {
94222f64 XL	254	ty::Int(_) => fx.bcx.ins().ineg(lane),
	255	ty::Float(_) => fx.bcx.ins().fneg(lane),
	256	_ => unreachable!(),
5e7ed085	257	}
94222f64 XL	258	});
	259	};
	260
5e7ed085 FG	261	simd_add \| simd_sub \| simd_mul \| simd_div \| simd_rem
	262	\| simd_shl \| simd_shr \| simd_and \| simd_or \| simd_xor, (c x, c y) {
	263	if !x.layout().ty.is_simd() {
	264	report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty);
	265	return;
	266	}
94222f64	267
5e7ed085 FG	268	// FIXME use vector instructions when possible
	269	simd_pair_for_each_lane(fx, x, y, ret, &\|fx, lane_ty, _ret_lane_ty, x_lane, y_lane\| match (
	270	lane_ty.kind(),
	271	intrinsic,
	272	) {
	273	(ty::Uint(_), sym::simd_add) => fx.bcx.ins().iadd(x_lane, y_lane),
	274	(ty::Uint(_), sym::simd_sub) => fx.bcx.ins().isub(x_lane, y_lane),
	275	(ty::Uint(_), sym::simd_mul) => fx.bcx.ins().imul(x_lane, y_lane),
	276	(ty::Uint(_), sym::simd_div) => fx.bcx.ins().udiv(x_lane, y_lane),
	277	(ty::Uint(_), sym::simd_rem) => fx.bcx.ins().urem(x_lane, y_lane),
	278
	279	(ty::Int(_), sym::simd_add) => fx.bcx.ins().iadd(x_lane, y_lane),
	280	(ty::Int(_), sym::simd_sub) => fx.bcx.ins().isub(x_lane, y_lane),
	281	(ty::Int(_), sym::simd_mul) => fx.bcx.ins().imul(x_lane, y_lane),
	282	(ty::Int(_), sym::simd_div) => fx.bcx.ins().sdiv(x_lane, y_lane),
	283	(ty::Int(_), sym::simd_rem) => fx.bcx.ins().srem(x_lane, y_lane),
	284
	285	(ty::Float(_), sym::simd_add) => fx.bcx.ins().fadd(x_lane, y_lane),
	286	(ty::Float(_), sym::simd_sub) => fx.bcx.ins().fsub(x_lane, y_lane),
	287	(ty::Float(_), sym::simd_mul) => fx.bcx.ins().fmul(x_lane, y_lane),
	288	(ty::Float(_), sym::simd_div) => fx.bcx.ins().fdiv(x_lane, y_lane),
	289	(ty::Float(FloatTy::F32), sym::simd_rem) => fx.lib_call(
	290	"fmodf",
	291	vec![AbiParam::new(types::F32), AbiParam::new(types::F32)],
	292	vec![AbiParam::new(types::F32)],
	293	&[x_lane, y_lane],
	294	)[0],
	295	(ty::Float(FloatTy::F64), sym::simd_rem) => fx.lib_call(
	296	"fmod",
	297	vec![AbiParam::new(types::F64), AbiParam::new(types::F64)],
	298	vec![AbiParam::new(types::F64)],
	299	&[x_lane, y_lane],
	300	)[0],
	301
	302	(ty::Uint(_), sym::simd_shl) => fx.bcx.ins().ishl(x_lane, y_lane),
	303	(ty::Uint(_), sym::simd_shr) => fx.bcx.ins().ushr(x_lane, y_lane),
	304	(ty::Uint(_), sym::simd_and) => fx.bcx.ins().band(x_lane, y_lane),
	305	(ty::Uint(_), sym::simd_or) => fx.bcx.ins().bor(x_lane, y_lane),
	306	(ty::Uint(_), sym::simd_xor) => fx.bcx.ins().bxor(x_lane, y_lane),
	307
	308	(ty::Int(_), sym::simd_shl) => fx.bcx.ins().ishl(x_lane, y_lane),
	309	(ty::Int(_), sym::simd_shr) => fx.bcx.ins().sshr(x_lane, y_lane),
	310	(ty::Int(_), sym::simd_and) => fx.bcx.ins().band(x_lane, y_lane),
	311	(ty::Int(_), sym::simd_or) => fx.bcx.ins().bor(x_lane, y_lane),
	312	(ty::Int(_), sym::simd_xor) => fx.bcx.ins().bxor(x_lane, y_lane),
	313
	314	_ => unreachable!(),
94222f64 XL	315	});
	316	};
	317
29967ef6	318	simd_fma, (c a, c b, c c) {
5e7ed085 FG	319	if !a.layout().ty.is_simd() {
	320	report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
	321	return;
	322	}
29967ef6 XL	323	assert_eq!(a.layout(), b.layout());
	324	assert_eq!(a.layout(), c.layout());
	325	let layout = a.layout();
	326
5869c6ff XL	327	let (lane_count, _lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
5869c6ff XL	328	let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
29967ef6	329	assert_eq!(lane_count, ret_lane_count);
5869c6ff	330	let ret_lane_layout = fx.layout_of(ret_lane_ty);
29967ef6 XL	331
29967ef6 XL	332	for lane in 0..lane_count {
94222f64 XL	333	let a_lane = a.value_lane(fx, lane).load_scalar(fx);
	334	let b_lane = b.value_lane(fx, lane).load_scalar(fx);
	335	let c_lane = c.value_lane(fx, lane).load_scalar(fx);
29967ef6 XL	336
	337	let mul_lane = fx.bcx.ins().fmul(a_lane, b_lane);
	338	let res_lane = CValue::by_val(fx.bcx.ins().fadd(mul_lane, c_lane), ret_lane_layout);
	339
94222f64	340	ret.place_lane(fx, lane).write_cvalue(fx, res_lane);
29967ef6 XL	341	}
	342	};
	343
5e7ed085 FG	344	simd_fmin \| simd_fmax, (c x, c y) {
	345	if !x.layout().ty.is_simd() {
	346	report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty);
	347	return;
	348	}
	349
	350	// FIXME use vector instructions when possible
	351	simd_pair_for_each_lane(fx, x, y, ret, &\|fx, lane_ty, _ret_lane_ty, x_lane, y_lane\| {
	352	match lane_ty.kind() {
	353	ty::Float(_) => {},
	354	_ => unreachable!("{:?}", lane_ty),
	355	}
	356	match intrinsic {
	357	sym::simd_fmin => fx.bcx.ins().fmin(x_lane, y_lane),
	358	sym::simd_fmax => fx.bcx.ins().fmax(x_lane, y_lane),
	359	_ => unreachable!(),
	360	}
	361	});
29967ef6 XL	362	};
29967ef6 XL	363
94222f64	364	simd_round, (c a) {
5e7ed085 FG	365	if !a.layout().ty.is_simd() {
	366	report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
	367	return;
	368	}
	369
	370	simd_for_each_lane(fx, a, ret, &\|fx, lane_ty, _ret_lane_ty, lane\| {
	371	match lane_ty.kind() {
94222f64 XL	372	ty::Float(FloatTy::F32) => fx.lib_call(
	373	"roundf",
	374	vec![AbiParam::new(types::F32)],
	375	vec![AbiParam::new(types::F32)],
	376	&[lane],
	377	)[0],
	378	ty::Float(FloatTy::F64) => fx.lib_call(
	379	"round",
	380	vec![AbiParam::new(types::F64)],
	381	vec![AbiParam::new(types::F64)],
	382	&[lane],
	383	)[0],
5e7ed085 FG	384	_ => unreachable!("{:?}", lane_ty),
5e7ed085 FG	385	}
94222f64 XL	386	});
94222f64 XL	387	};
5e7ed085 FG	388
	389	simd_fabs \| simd_fsqrt \| simd_ceil \| simd_floor \| simd_trunc, (c a) {
	390	if !a.layout().ty.is_simd() {
	391	report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
	392	return;
	393	}
	394
	395	simd_for_each_lane(fx, a, ret, &\|fx, lane_ty, _ret_lane_ty, lane\| {
	396	match lane_ty.kind() {
	397	ty::Float(_) => {},
	398	_ => unreachable!("{:?}", lane_ty),
	399	}
	400	match intrinsic {
	401	sym::simd_fabs => fx.bcx.ins().fabs(lane),
	402	sym::simd_fsqrt => fx.bcx.ins().sqrt(lane),
	403	sym::simd_ceil => fx.bcx.ins().ceil(lane),
	404	sym::simd_floor => fx.bcx.ins().floor(lane),
	405	sym::simd_trunc => fx.bcx.ins().trunc(lane),
	406	_ => unreachable!(),
	407	}
94222f64 XL	408	});
	409	};
	410
	411	simd_reduce_add_ordered \| simd_reduce_add_unordered, (c v, v acc) {
5e7ed085 FG	412	// FIXME there must be no acc param for integer vectors
	413	if !v.layout().ty.is_simd() {
	414	report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
	415	return;
	416	}
	417
	418	simd_reduce(fx, v, Some(acc), ret, &\|fx, lane_ty, a, b\| {
	419	if lane_ty.is_floating_point() {
fc512014 XL	420	fx.bcx.ins().fadd(a, b)
	421	} else {
	422	fx.bcx.ins().iadd(a, b)
	423	}
	424	});
	425	};
	426
94222f64	427	simd_reduce_mul_ordered \| simd_reduce_mul_unordered, (c v, v acc) {
5e7ed085 FG	428	// FIXME there must be no acc param for integer vectors
	429	if !v.layout().ty.is_simd() {
	430	report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
	431	return;
	432	}
	433
	434	simd_reduce(fx, v, Some(acc), ret, &\|fx, lane_ty, a, b\| {
	435	if lane_ty.is_floating_point() {
fc512014 XL	436	fx.bcx.ins().fmul(a, b)
	437	} else {
	438	fx.bcx.ins().imul(a, b)
	439	}
	440	});
	441	};
	442
	443	simd_reduce_all, (c v) {
5e7ed085 FG	444	if !v.layout().ty.is_simd() {
	445	report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
	446	return;
	447	}
	448
	449	simd_reduce_bool(fx, v, ret, &\|fx, a, b\| fx.bcx.ins().band(a, b));
fc512014 XL	450	};
	451
	452	simd_reduce_any, (c v) {
5e7ed085 FG	453	if !v.layout().ty.is_simd() {
	454	report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
	455	return;
	456	}
	457
	458	simd_reduce_bool(fx, v, ret, &\|fx, a, b\| fx.bcx.ins().bor(a, b));
fc512014 XL	459	};
fc512014 XL	460
94222f64	461	simd_reduce_and, (c v) {
5e7ed085 FG	462	if !v.layout().ty.is_simd() {
	463	report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
	464	return;
	465	}
	466
	467	simd_reduce(fx, v, None, ret, &\|fx, _ty, a, b\| fx.bcx.ins().band(a, b));
94222f64 XL	468	};
	469
	470	simd_reduce_or, (c v) {
5e7ed085 FG	471	if !v.layout().ty.is_simd() {
	472	report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
	473	return;
	474	}
	475
	476	simd_reduce(fx, v, None, ret, &\|fx, _ty, a, b\| fx.bcx.ins().bor(a, b));
94222f64 XL	477	};
	478
	479	simd_reduce_xor, (c v) {
5e7ed085 FG	480	if !v.layout().ty.is_simd() {
	481	report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
	482	return;
	483	}
	484
	485	simd_reduce(fx, v, None, ret, &\|fx, _ty, a, b\| fx.bcx.ins().bxor(a, b));
94222f64 XL	486	};
	487
	488	simd_reduce_min, (c v) {
5e7ed085 FG	489	if !v.layout().ty.is_simd() {
	490	report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
	491	return;
	492	}
	493
	494	simd_reduce(fx, v, None, ret, &\|fx, ty, a, b\| {
	495	let lt = match ty.kind() {
a2a8927a XL	496	ty::Int(_) => fx.bcx.ins().icmp(IntCC::SignedLessThan, a, b),
	497	ty::Uint(_) => fx.bcx.ins().icmp(IntCC::UnsignedLessThan, a, b),
	498	ty::Float(_) => fx.bcx.ins().fcmp(FloatCC::LessThan, a, b),
	499	_ => unreachable!(),
	500	};
94222f64 XL	501	fx.bcx.ins().select(lt, a, b)
	502	});
	503	};
	504
	505	simd_reduce_max, (c v) {
5e7ed085 FG	506	if !v.layout().ty.is_simd() {
	507	report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
	508	return;
	509	}
	510
	511	simd_reduce(fx, v, None, ret, &\|fx, ty, a, b\| {
	512	let gt = match ty.kind() {
a2a8927a XL	513	ty::Int(_) => fx.bcx.ins().icmp(IntCC::SignedGreaterThan, a, b),
	514	ty::Uint(_) => fx.bcx.ins().icmp(IntCC::UnsignedGreaterThan, a, b),
	515	ty::Float(_) => fx.bcx.ins().fcmp(FloatCC::GreaterThan, a, b),
	516	_ => unreachable!(),
	517	};
94222f64 XL	518	fx.bcx.ins().select(gt, a, b)
	519	});
	520	};
	521
	522	simd_select, (c m, c a, c b) {
5e7ed085 FG	523	if !m.layout().ty.is_simd() {
	524	report_simd_type_validation_error(fx, intrinsic, span, m.layout().ty);
	525	return;
	526	}
	527	if !a.layout().ty.is_simd() {
	528	report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
	529	return;
	530	}
94222f64 XL	531	assert_eq!(a.layout(), b.layout());
	532
	533	let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx);
	534	let lane_layout = fx.layout_of(lane_ty);
	535
	536	for lane in 0..lane_count {
	537	let m_lane = m.value_lane(fx, lane).load_scalar(fx);
	538	let a_lane = a.value_lane(fx, lane).load_scalar(fx);
	539	let b_lane = b.value_lane(fx, lane).load_scalar(fx);
	540
	541	let m_lane = fx.bcx.ins().icmp_imm(IntCC::Equal, m_lane, 0);
	542	let res_lane = CValue::by_val(fx.bcx.ins().select(m_lane, b_lane, a_lane), lane_layout);
	543
	544	ret.place_lane(fx, lane).write_cvalue(fx, res_lane);
	545	}
	546	};
	547
	548	// simd_saturating_*
29967ef6	549	// simd_bitmask
94222f64 XL	550	// simd_scatter
94222f64 XL	551	// simd_gather
29967ef6 XL	552	}
29967ef6 XL	553	}