compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs

   1 //! Emulate LLVM intrinsics
   2
   3 use crate::intrinsics::*;
   4 use crate::prelude::*;
   5
   6 use rustc_middle::ty::subst::SubstsRef;
   7
   8 pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
   9     fx: &mut FunctionCx<'_, '_, 'tcx>,
  10     intrinsic: &str,
  11     substs: SubstsRef<'tcx>,
  12     args: &[mir::Operand<'tcx>],
  13     destination: Option<(CPlace<'tcx>, BasicBlock)>,
  14 ) {
  15     let ret = destination.unwrap().0;
  16
  17     intrinsic_match! {
  18         fx, intrinsic, substs, args,
  19         _ => {
  20             fx.tcx.sess.warn(&format!("unsupported llvm intrinsic {}; replacing with trap", intrinsic));
  21             crate::trap::trap_unimplemented(fx, intrinsic);
  22         };
  23
  24         // Used by `_mm_movemask_epi8` and `_mm256_movemask_epi8`
  25         llvm.x86.sse2.pmovmskb.128 | llvm.x86.avx2.pmovmskb | llvm.x86.sse2.movmsk.pd, (c a) {
  26             let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx);
  27             let lane_ty = fx.clif_type(lane_ty).unwrap();
  28             assert!(lane_count <= 32);
  29
  30             let mut res = fx.bcx.ins().iconst(types::I32, 0);
  31
  32             for lane in (0..lane_count).rev() {
  33                 let a_lane = a.value_field(fx, mir::Field::new(lane.try_into().unwrap())).load_scalar(fx);
  34
  35                 // cast float to int
  36                 let a_lane = match lane_ty {
  37                     types::F32 => fx.bcx.ins().bitcast(types::I32, a_lane),
  38                     types::F64 => fx.bcx.ins().bitcast(types::I64, a_lane),
  39                     _ => a_lane,
  40                 };
  41
  42                 // extract sign bit of an int
  43                 let a_lane_sign = fx.bcx.ins().ushr_imm(a_lane, i64::from(lane_ty.bits() - 1));
  44
  45                 // shift sign bit into result
  46                 let a_lane_sign = clif_intcast(fx, a_lane_sign, types::I32, false);
  47                 res = fx.bcx.ins().ishl_imm(res, 1);
  48                 res = fx.bcx.ins().bor(res, a_lane_sign);
  49             }
  50
  51             let res = CValue::by_val(res, fx.layout_of(fx.tcx.types.i32));
  52             ret.write_cvalue(fx, res);
  53         };
  54         llvm.x86.sse2.cmp.ps | llvm.x86.sse2.cmp.pd, (c x, c y, o kind) {
  55             let kind_const = crate::constant::mir_operand_get_const_val(fx, kind).expect("llvm.x86.sse2.cmp.* kind not const");
  56             let flt_cc = match kind_const.try_to_bits(Size::from_bytes(1)).unwrap_or_else(|| panic!("kind not scalar: {:?}", kind_const)) {
  57                 0 => FloatCC::Equal,
  58                 1 => FloatCC::LessThan,
  59                 2 => FloatCC::LessThanOrEqual,
  60                 7 => {
  61                     unimplemented!("Compares corresponding elements in `a` and `b` to see if neither is `NaN`.");
  62                 }
  63                 3 => {
  64                     unimplemented!("Compares corresponding elements in `a` and `b` to see if either is `NaN`.");
  65                 }
  66                 4 => FloatCC::NotEqual,
  67                 5 => {
  68                     unimplemented!("not less than");
  69                 }
  70                 6 => {
  71                     unimplemented!("not less than or equal");
  72                 }
  73                 kind => unreachable!("kind {:?}", kind),
  74             };
  75
  76             simd_pair_for_each_lane(fx, x, y, ret, |fx, lane_layout, res_lane_layout, x_lane, y_lane| {
  77                 let res_lane = match lane_layout.ty.kind() {
  78                     ty::Float(_) => fx.bcx.ins().fcmp(flt_cc, x_lane, y_lane),
  79                     _ => unreachable!("{:?}", lane_layout.ty),
  80                 };
  81                 bool_to_zero_or_max_uint(fx, res_lane_layout, res_lane)
  82             });
  83         };
  84         llvm.x86.sse2.psrli.d, (c a, o imm8) {
  85             let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8).expect("llvm.x86.sse2.psrli.d imm8 not const");
  86             simd_for_each_lane(fx, a, ret, |fx, _lane_layout, res_lane_layout, lane| {
  87                 let res_lane = match imm8.try_to_bits(Size::from_bytes(4)).unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) {
  88                     imm8 if imm8 < 32 => fx.bcx.ins().ushr_imm(lane, i64::from(imm8 as u8)),
  89                     _ => fx.bcx.ins().iconst(types::I32, 0),
  90                 };
  91                 CValue::by_val(res_lane, res_lane_layout)
  92             });
  93         };
  94         llvm.x86.sse2.pslli.d, (c a, o imm8) {
  95             let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8).expect("llvm.x86.sse2.psrli.d imm8 not const");
  96             simd_for_each_lane(fx, a, ret, |fx, _lane_layout, res_lane_layout, lane| {
  97                 let res_lane = match imm8.try_to_bits(Size::from_bytes(4)).unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) {
  98                     imm8 if imm8 < 32 => fx.bcx.ins().ishl_imm(lane, i64::from(imm8 as u8)),
  99                     _ => fx.bcx.ins().iconst(types::I32, 0),
 100                 };
 101                 CValue::by_val(res_lane, res_lane_layout)
 102             });
 103         };
 104         llvm.x86.sse2.storeu.dq, (v mem_addr, c a) {
 105             // FIXME correctly handle the unalignment
 106             let dest = CPlace::for_ptr(Pointer::new(mem_addr), a.layout());
 107             dest.write_cvalue(fx, a);
 108         };
 109     }
 110
 111     if let Some((_, dest)) = destination {
 112         let ret_block = fx.get_block(dest);
 113         fx.bcx.ins().jump(ret_block, &[]);
 114     } else {
 115         trap_unreachable(fx, "[corruption] Diverging intrinsic returned.");
 116     }
 117 }
 118
 119 // llvm.x86.avx2.vperm2i128
 120 // llvm.x86.ssse3.pshuf.b.128
 121 // llvm.x86.avx2.pshuf.b
 122 // llvm.x86.avx2.psrli.w
 123 // llvm.x86.sse2.psrli.w