compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs

   1 //! Codegen of intrinsics. This includes `extern "rust-intrinsic"`, `extern "platform-intrinsic"`
   2 //! and LLVM intrinsics that have symbol names starting with `llvm.`.
   3
   4 mod cpuid;
   5 mod llvm;
   6 mod simd;
   7
   8 pub(crate) use cpuid::codegen_cpuid_call;
   9 pub(crate) use llvm::codegen_llvm_intrinsic_call;
  10
  11 use crate::prelude::*;
  12 use rustc_middle::ty::print::with_no_trimmed_paths;
  13
  14 macro intrinsic_pat {
  15     (_) => {
  16         _
  17     },
  18     ($name:ident) => {
  19         stringify!($name)
  20     },
  21     ($name:literal) => {
  22         stringify!($name)
  23     },
  24     ($x:ident . $($xs:tt).*) => {
  25         concat!(stringify!($x), ".", intrinsic_pat!($($xs).*))
  26     }
  27 }
  28
  29 macro intrinsic_arg {
  30     (o $fx:expr, $arg:ident) => {
  31         $arg
  32     },
  33     (c $fx:expr, $arg:ident) => {
  34         codegen_operand($fx, $arg)
  35     },
  36     (v $fx:expr, $arg:ident) => {
  37         codegen_operand($fx, $arg).load_scalar($fx)
  38     }
  39 }
  40
  41 macro intrinsic_substs {
  42     ($substs:expr, $index:expr,) => {},
  43     ($substs:expr, $index:expr, $first:ident $(,$rest:ident)*) => {
  44         let $first = $substs.type_at($index);
  45         intrinsic_substs!($substs, $index+1, $($rest),*);
  46     }
  47 }
  48
  49 macro intrinsic_match {
  50     ($fx:expr, $intrinsic:expr, $substs:expr, $args:expr,
  51     _ => $unknown:block;
  52     $(
  53         $($($name:tt).*)|+ $(if $cond:expr)?, $(<$($subst:ident),*>)? ($($a:ident $arg:ident),*) $content:block;
  54     )*) => {
  55         let _ = $substs; // Silence warning when substs is unused.
  56         match $intrinsic {
  57             $(
  58                 $(intrinsic_pat!($($name).*))|* $(if $cond)? => {
  59                     #[allow(unused_parens, non_snake_case)]
  60                     {
  61                         $(
  62                             intrinsic_substs!($substs, 0, $($subst),*);
  63                         )?
  64                         if let [$($arg),*] = $args {
  65                             let ($($arg,)*) = (
  66                                 $(intrinsic_arg!($a $fx, $arg),)*
  67                             );
  68                             #[warn(unused_parens, non_snake_case)]
  69                             {
  70                                 $content
  71                             }
  72                         } else {
  73                             bug!("wrong number of args for intrinsic {:?}", $intrinsic);
  74                         }
  75                     }
  76                 }
  77             )*
  78             _ => $unknown,
  79         }
  80     }
  81 }
  82
  83 macro call_intrinsic_match {
  84     ($fx:expr, $intrinsic:expr, $substs:expr, $ret:expr, $destination:expr, $args:expr, $(
  85         $name:ident($($arg:ident),*) -> $ty:ident => $func:ident,
  86     )*) => {
  87         match $intrinsic {
  88             $(
  89                 stringify!($name) => {
  90                     assert!($substs.is_noop());
  91                     if let [$(ref $arg),*] = *$args {
  92                         let ($($arg,)*) = (
  93                             $(codegen_operand($fx, $arg),)*
  94                         );
  95                         let res = $fx.easy_call(stringify!($func), &[$($arg),*], $fx.tcx.types.$ty);
  96                         $ret.write_cvalue($fx, res);
  97
  98                         if let Some((_, dest)) = $destination {
  99                             let ret_block = $fx.get_block(dest);
 100                             $fx.bcx.ins().jump(ret_block, &[]);
 101                             return;
 102                         } else {
 103                             unreachable!();
 104                         }
 105                     } else {
 106                         bug!("wrong number of args for intrinsic {:?}", $intrinsic);
 107                     }
 108                 }
 109             )*
 110             _ => {}
 111         }
 112     }
 113 }
 114
 115 macro atomic_binop_return_old($fx:expr, $op:ident<$T:ident>($ptr:ident, $src:ident) -> $ret:ident) {
 116     crate::atomic_shim::lock_global_lock($fx);
 117
 118     let clif_ty = $fx.clif_type($T).unwrap();
 119     let old = $fx.bcx.ins().load(clif_ty, MemFlags::new(), $ptr, 0);
 120     let new = $fx.bcx.ins().$op(old, $src);
 121     $fx.bcx.ins().store(MemFlags::new(), new, $ptr, 0);
 122     $ret.write_cvalue($fx, CValue::by_val(old, $fx.layout_of($T)));
 123
 124     crate::atomic_shim::unlock_global_lock($fx);
 125 }
 126
 127 macro atomic_minmax($fx:expr, $cc:expr, <$T:ident> ($ptr:ident, $src:ident) -> $ret:ident) {
 128     crate::atomic_shim::lock_global_lock($fx);
 129
 130     // Read old
 131     let clif_ty = $fx.clif_type($T).unwrap();
 132     let old = $fx.bcx.ins().load(clif_ty, MemFlags::new(), $ptr, 0);
 133
 134     // Compare
 135     let is_eq = $fx.bcx.ins().icmp(IntCC::SignedGreaterThan, old, $src);
 136     let new = $fx.bcx.ins().select(is_eq, old, $src);
 137
 138     // Write new
 139     $fx.bcx.ins().store(MemFlags::new(), new, $ptr, 0);
 140
 141     let ret_val = CValue::by_val(old, $ret.layout());
 142     $ret.write_cvalue($fx, ret_val);
 143
 144     crate::atomic_shim::unlock_global_lock($fx);
 145 }
 146
 147 macro validate_atomic_type($fx:ident, $intrinsic:ident, $span:ident, $ty:expr) {
 148     match $ty.kind() {
 149         ty::Uint(_) | ty::Int(_) => {}
 150         _ => {
 151             $fx.tcx.sess.span_err(
 152                 $span,
 153                 &format!(
 154                     "`{}` intrinsic: expected basic integer type, found `{:?}`",
 155                     $intrinsic, $ty
 156                 ),
 157             );
 158             // Prevent verifier error
 159             crate::trap::trap_unreachable($fx, "compilation should not have succeeded");
 160             return;
 161         }
 162     }
 163 }
 164
 165 macro validate_simd_type($fx:ident, $intrinsic:ident, $span:ident, $ty:expr) {
 166     if !$ty.is_simd() {
 167         $fx.tcx.sess.span_err($span, &format!("invalid monomorphization of `{}` intrinsic: expected SIMD input type, found non-SIMD `{}`", $intrinsic, $ty));
 168         // Prevent verifier error
 169         crate::trap::trap_unreachable($fx, "compilation should not have succeeded");
 170         return;
 171     }
 172 }
 173
 174 fn lane_type_and_count<'tcx>(
 175     tcx: TyCtxt<'tcx>,
 176     layout: TyAndLayout<'tcx>,
 177 ) -> (TyAndLayout<'tcx>, u16) {
 178     assert!(layout.ty.is_simd());
 179     let lane_count = match layout.fields {
 180         rustc_target::abi::FieldsShape::Array { stride: _, count } => u16::try_from(count).unwrap(),
 181         _ => unreachable!("lane_type_and_count({:?})", layout),
 182     };
 183     let lane_layout = layout
 184         .field(
 185             &ty::layout::LayoutCx {
 186                 tcx,
 187                 param_env: ParamEnv::reveal_all(),
 188             },
 189             0,
 190         )
 191         .unwrap();
 192     (lane_layout, lane_count)
 193 }
 194
 195 pub(crate) fn clif_vector_type<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx>) -> Option<Type> {
 196     let (element, count) = match &layout.abi {
 197         Abi::Vector { element, count } => (element.clone(), *count),
 198         _ => unreachable!(),
 199     };
 200
 201     match scalar_to_clif_type(tcx, element).by(u16::try_from(count).unwrap()) {
 202         // Cranelift currently only implements icmp for 128bit vectors.
 203         Some(vector_ty) if vector_ty.bits() == 128 => Some(vector_ty),
 204         _ => None,
 205     }
 206 }
 207
 208 fn simd_for_each_lane<'tcx, M: Module>(
 209     fx: &mut FunctionCx<'_, 'tcx, M>,
 210     val: CValue<'tcx>,
 211     ret: CPlace<'tcx>,
 212     f: impl Fn(
 213         &mut FunctionCx<'_, 'tcx, M>,
 214         TyAndLayout<'tcx>,
 215         TyAndLayout<'tcx>,
 216         Value,
 217     ) -> CValue<'tcx>,
 218 ) {
 219     let layout = val.layout();
 220
 221     let (lane_layout, lane_count) = lane_type_and_count(fx.tcx, layout);
 222     let (ret_lane_layout, ret_lane_count) = lane_type_and_count(fx.tcx, ret.layout());
 223     assert_eq!(lane_count, ret_lane_count);
 224
 225     for lane_idx in 0..lane_count {
 226         let lane_idx = mir::Field::new(lane_idx.try_into().unwrap());
 227         let lane = val.value_field(fx, lane_idx).load_scalar(fx);
 228
 229         let res_lane = f(fx, lane_layout, ret_lane_layout, lane);
 230
 231         ret.place_field(fx, lane_idx).write_cvalue(fx, res_lane);
 232     }
 233 }
 234
 235 fn simd_pair_for_each_lane<'tcx, M: Module>(
 236     fx: &mut FunctionCx<'_, 'tcx, M>,
 237     x: CValue<'tcx>,
 238     y: CValue<'tcx>,
 239     ret: CPlace<'tcx>,
 240     f: impl Fn(
 241         &mut FunctionCx<'_, 'tcx, M>,
 242         TyAndLayout<'tcx>,
 243         TyAndLayout<'tcx>,
 244         Value,
 245         Value,
 246     ) -> CValue<'tcx>,
 247 ) {
 248     assert_eq!(x.layout(), y.layout());
 249     let layout = x.layout();
 250
 251     let (lane_layout, lane_count) = lane_type_and_count(fx.tcx, layout);
 252     let (ret_lane_layout, ret_lane_count) = lane_type_and_count(fx.tcx, ret.layout());
 253     assert_eq!(lane_count, ret_lane_count);
 254
 255     for lane in 0..lane_count {
 256         let lane = mir::Field::new(lane.try_into().unwrap());
 257         let x_lane = x.value_field(fx, lane).load_scalar(fx);
 258         let y_lane = y.value_field(fx, lane).load_scalar(fx);
 259
 260         let res_lane = f(fx, lane_layout, ret_lane_layout, x_lane, y_lane);
 261
 262         ret.place_field(fx, lane).write_cvalue(fx, res_lane);
 263     }
 264 }
 265
 266 fn bool_to_zero_or_max_uint<'tcx>(
 267     fx: &mut FunctionCx<'_, 'tcx, impl Module>,
 268     layout: TyAndLayout<'tcx>,
 269     val: Value,
 270 ) -> CValue<'tcx> {
 271     let ty = fx.clif_type(layout.ty).unwrap();
 272
 273     let int_ty = match ty {
 274         types::F32 => types::I32,
 275         types::F64 => types::I64,
 276         ty => ty,
 277     };
 278
 279     let val = fx.bcx.ins().bint(int_ty, val);
 280     let mut res = fx.bcx.ins().ineg(val);
 281
 282     if ty.is_float() {
 283         res = fx.bcx.ins().bitcast(ty, res);
 284     }
 285
 286     CValue::by_val(res, layout)
 287 }
 288
 289 macro simd_cmp {
 290     ($fx:expr, $cc:ident($x:ident, $y:ident) -> $ret:ident) => {
 291         let vector_ty = clif_vector_type($fx.tcx, $x.layout());
 292
 293         if let Some(vector_ty) = vector_ty {
 294             let x = $x.load_scalar($fx);
 295             let y = $y.load_scalar($fx);
 296             let val = $fx.bcx.ins().icmp(IntCC::$cc, x, y);
 297
 298             // HACK This depends on the fact that icmp for vectors represents bools as 0 and !0, not 0 and 1.
 299             let val = $fx.bcx.ins().raw_bitcast(vector_ty, val);
 300
 301             $ret.write_cvalue($fx, CValue::by_val(val, $ret.layout()));
 302         } else {
 303             simd_pair_for_each_lane(
 304                 $fx,
 305                 $x,
 306                 $y,
 307                 $ret,
 308                 |fx, lane_layout, res_lane_layout, x_lane, y_lane| {
 309                     let res_lane = match lane_layout.ty.kind() {
 310                         ty::Uint(_) | ty::Int(_) => fx.bcx.ins().icmp(IntCC::$cc, x_lane, y_lane),
 311                         _ => unreachable!("{:?}", lane_layout.ty),
 312                     };
 313                     bool_to_zero_or_max_uint(fx, res_lane_layout, res_lane)
 314                 },
 315             );
 316         }
 317     },
 318     ($fx:expr, $cc_u:ident|$cc_s:ident($x:ident, $y:ident) -> $ret:ident) => {
 319         // FIXME use vector icmp when possible
 320         simd_pair_for_each_lane(
 321             $fx,
 322             $x,
 323             $y,
 324             $ret,
 325             |fx, lane_layout, res_lane_layout, x_lane, y_lane| {
 326                 let res_lane = match lane_layout.ty.kind() {
 327                     ty::Uint(_) => fx.bcx.ins().icmp(IntCC::$cc_u, x_lane, y_lane),
 328                     ty::Int(_) => fx.bcx.ins().icmp(IntCC::$cc_s, x_lane, y_lane),
 329                     _ => unreachable!("{:?}", lane_layout.ty),
 330                 };
 331                 bool_to_zero_or_max_uint(fx, res_lane_layout, res_lane)
 332             },
 333         );
 334     },
 335 }
 336
 337 macro simd_int_binop {
 338     ($fx:expr, $op:ident($x:ident, $y:ident) -> $ret:ident) => {
 339         simd_int_binop!($fx, $op|$op($x, $y) -> $ret);
 340     },
 341     ($fx:expr, $op_u:ident|$op_s:ident($x:ident, $y:ident) -> $ret:ident) => {
 342         simd_pair_for_each_lane(
 343             $fx,
 344             $x,
 345             $y,
 346             $ret,
 347             |fx, lane_layout, ret_lane_layout, x_lane, y_lane| {
 348                 let res_lane = match lane_layout.ty.kind() {
 349                     ty::Uint(_) => fx.bcx.ins().$op_u(x_lane, y_lane),
 350                     ty::Int(_) => fx.bcx.ins().$op_s(x_lane, y_lane),
 351                     _ => unreachable!("{:?}", lane_layout.ty),
 352                 };
 353                 CValue::by_val(res_lane, ret_lane_layout)
 354             },
 355         );
 356     },
 357 }
 358
 359 macro simd_int_flt_binop {
 360     ($fx:expr, $op:ident|$op_f:ident($x:ident, $y:ident) -> $ret:ident) => {
 361         simd_int_flt_binop!($fx, $op|$op|$op_f($x, $y) -> $ret);
 362     },
 363     ($fx:expr, $op_u:ident|$op_s:ident|$op_f:ident($x:ident, $y:ident) -> $ret:ident) => {
 364         simd_pair_for_each_lane(
 365             $fx,
 366             $x,
 367             $y,
 368             $ret,
 369             |fx, lane_layout, ret_lane_layout, x_lane, y_lane| {
 370                 let res_lane = match lane_layout.ty.kind() {
 371                     ty::Uint(_) => fx.bcx.ins().$op_u(x_lane, y_lane),
 372                     ty::Int(_) => fx.bcx.ins().$op_s(x_lane, y_lane),
 373                     ty::Float(_) => fx.bcx.ins().$op_f(x_lane, y_lane),
 374                     _ => unreachable!("{:?}", lane_layout.ty),
 375                 };
 376                 CValue::by_val(res_lane, ret_lane_layout)
 377             },
 378         );
 379     },
 380 }
 381
 382 macro simd_flt_binop($fx:expr, $op:ident($x:ident, $y:ident) -> $ret:ident) {
 383     simd_pair_for_each_lane(
 384         $fx,
 385         $x,
 386         $y,
 387         $ret,
 388         |fx, lane_layout, ret_lane_layout, x_lane, y_lane| {
 389             let res_lane = match lane_layout.ty.kind() {
 390                 ty::Float(_) => fx.bcx.ins().$op(x_lane, y_lane),
 391                 _ => unreachable!("{:?}", lane_layout.ty),
 392             };
 393             CValue::by_val(res_lane, ret_lane_layout)
 394         },
 395     );
 396 }
 397
 398 pub(crate) fn codegen_intrinsic_call<'tcx>(
 399     fx: &mut FunctionCx<'_, 'tcx, impl Module>,
 400     instance: Instance<'tcx>,
 401     args: &[mir::Operand<'tcx>],
 402     destination: Option<(CPlace<'tcx>, BasicBlock)>,
 403     span: Span,
 404 ) {
 405     let def_id = instance.def_id();
 406     let substs = instance.substs;
 407
 408     let intrinsic = fx.tcx.item_name(def_id).as_str();
 409     let intrinsic = &intrinsic[..];
 410
 411     let ret = match destination {
 412         Some((place, _)) => place,
 413         None => {
 414             // Insert non returning intrinsics here
 415             match intrinsic {
 416                 "abort" => {
 417                     trap_abort(fx, "Called intrinsic::abort.");
 418                 }
 419                 "unreachable" => {
 420                     trap_unreachable(fx, "[corruption] Called intrinsic::unreachable.");
 421                 }
 422                 "transmute" => {
 423                     crate::base::codegen_panic(fx, "Transmuting to uninhabited type.", span);
 424                 }
 425                 _ => unimplemented!("unsupported instrinsic {}", intrinsic),
 426             }
 427             return;
 428         }
 429     };
 430
 431     if intrinsic.starts_with("simd_") {
 432         self::simd::codegen_simd_intrinsic_call(fx, instance, args, ret, span);
 433         let ret_block = fx.get_block(destination.expect("SIMD intrinsics don't diverge").1);
 434         fx.bcx.ins().jump(ret_block, &[]);
 435         return;
 436     }
 437
 438     let usize_layout = fx.layout_of(fx.tcx.types.usize);
 439
 440     call_intrinsic_match! {
 441         fx, intrinsic, substs, ret, destination, args,
 442         expf32(flt) -> f32 => expf,
 443         expf64(flt) -> f64 => exp,
 444         exp2f32(flt) -> f32 => exp2f,
 445         exp2f64(flt) -> f64 => exp2,
 446         sqrtf32(flt) -> f32 => sqrtf,
 447         sqrtf64(flt) -> f64 => sqrt,
 448         powif32(a, x) -> f32 => __powisf2, // compiler-builtins
 449         powif64(a, x) -> f64 => __powidf2, // compiler-builtins
 450         powf32(a, x) -> f32 => powf,
 451         powf64(a, x) -> f64 => pow,
 452         logf32(flt) -> f32 => logf,
 453         logf64(flt) -> f64 => log,
 454         log2f32(flt) -> f32 => log2f,
 455         log2f64(flt) -> f64 => log2,
 456         log10f32(flt) -> f32 => log10f,
 457         log10f64(flt) -> f64 => log10,
 458         fabsf32(flt) -> f32 => fabsf,
 459         fabsf64(flt) -> f64 => fabs,
 460         fmaf32(x, y, z) -> f32 => fmaf,
 461         fmaf64(x, y, z) -> f64 => fma,
 462         copysignf32(x, y) -> f32 => copysignf,
 463         copysignf64(x, y) -> f64 => copysign,
 464
 465         // rounding variants
 466         // FIXME use clif insts
 467         floorf32(flt) -> f32 => floorf,
 468         floorf64(flt) -> f64 => floor,
 469         ceilf32(flt) -> f32 => ceilf,
 470         ceilf64(flt) -> f64 => ceil,
 471         truncf32(flt) -> f32 => truncf,
 472         truncf64(flt) -> f64 => trunc,
 473         roundf32(flt) -> f32 => roundf,
 474         roundf64(flt) -> f64 => round,
 475
 476         // trigonometry
 477         sinf32(flt) -> f32 => sinf,
 478         sinf64(flt) -> f64 => sin,
 479         cosf32(flt) -> f32 => cosf,
 480         cosf64(flt) -> f64 => cos,
 481         tanf32(flt) -> f32 => tanf,
 482         tanf64(flt) -> f64 => tan,
 483     }
 484
 485     intrinsic_match! {
 486         fx, intrinsic, substs, args,
 487         _ => {
 488             fx.tcx.sess.span_fatal(span, &format!("unsupported intrinsic {}", intrinsic));
 489         };
 490
 491         assume, (c _a) {};
 492         likely | unlikely, (c a) {
 493             ret.write_cvalue(fx, a);
 494         };
 495         breakpoint, () {
 496             fx.bcx.ins().debugtrap();
 497         };
 498         copy | copy_nonoverlapping, <elem_ty> (v src, v dst, v count) {
 499             let elem_size: u64 = fx.layout_of(elem_ty).size.bytes();
 500             let elem_size = fx
 501                 .bcx
 502                 .ins()
 503                 .iconst(fx.pointer_type, elem_size as i64);
 504             assert_eq!(args.len(), 3);
 505             let byte_amount = fx.bcx.ins().imul(count, elem_size);
 506
 507             if intrinsic.contains("nonoverlapping") {
 508                 // FIXME emit_small_memcpy
 509                 fx.bcx.call_memcpy(fx.cx.module.target_config(), dst, src, byte_amount);
 510             } else {
 511                 // FIXME emit_small_memmove
 512                 fx.bcx.call_memmove(fx.cx.module.target_config(), dst, src, byte_amount);
 513             }
 514         };
 515         // NOTE: the volatile variants have src and dst swapped
 516         volatile_copy_memory | volatile_copy_nonoverlapping_memory, <elem_ty> (v dst, v src, v count) {
 517             let elem_size: u64 = fx.layout_of(elem_ty).size.bytes();
 518             let elem_size = fx
 519                 .bcx
 520                 .ins()
 521                 .iconst(fx.pointer_type, elem_size as i64);
 522             assert_eq!(args.len(), 3);
 523             let byte_amount = fx.bcx.ins().imul(count, elem_size);
 524
 525             // FIXME make the copy actually volatile when using emit_small_mem{cpy,move}
 526             if intrinsic.contains("nonoverlapping") {
 527                 // FIXME emit_small_memcpy
 528                 fx.bcx.call_memcpy(fx.cx.module.target_config(), dst, src, byte_amount);
 529             } else {
 530                 // FIXME emit_small_memmove
 531                 fx.bcx.call_memmove(fx.cx.module.target_config(), dst, src, byte_amount);
 532             }
 533         };
 534         discriminant_value, (c ptr) {
 535             let pointee_layout = fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty);
 536             let val = CValue::by_ref(Pointer::new(ptr.load_scalar(fx)), pointee_layout);
 537             let discr = crate::discriminant::codegen_get_discriminant(fx, val, ret.layout());
 538             ret.write_cvalue(fx, discr);
 539         };
 540         size_of_val, <T> (c ptr) {
 541             let layout = fx.layout_of(T);
 542             let size = if layout.is_unsized() {
 543                 let (_ptr, info) = ptr.load_scalar_pair(fx);
 544                 let (size, _align) = crate::unsize::size_and_align_of_dst(fx, layout, info);
 545                 size
 546             } else {
 547                 fx
 548                     .bcx
 549                     .ins()
 550                     .iconst(fx.pointer_type, layout.size.bytes() as i64)
 551             };
 552             ret.write_cvalue(fx, CValue::by_val(size, usize_layout));
 553         };
 554         min_align_of_val, <T> (c ptr) {
 555             let layout = fx.layout_of(T);
 556             let align = if layout.is_unsized() {
 557                 let (_ptr, info) = ptr.load_scalar_pair(fx);
 558                 let (_size, align) = crate::unsize::size_and_align_of_dst(fx, layout, info);
 559                 align
 560             } else {
 561                 fx
 562                     .bcx
 563                     .ins()
 564                     .iconst(fx.pointer_type, layout.align.abi.bytes() as i64)
 565             };
 566             ret.write_cvalue(fx, CValue::by_val(align, usize_layout));
 567         };
 568
 569         _ if intrinsic.starts_with("unchecked_") || intrinsic == "exact_div", (c x, c y) {
 570             // FIXME trap on overflow
 571             let bin_op = match intrinsic {
 572                 "unchecked_add" => BinOp::Add,
 573                 "unchecked_sub" => BinOp::Sub,
 574                 "unchecked_div" | "exact_div" => BinOp::Div,
 575                 "unchecked_rem" => BinOp::Rem,
 576                 "unchecked_shl" => BinOp::Shl,
 577                 "unchecked_shr" => BinOp::Shr,
 578                 _ => unreachable!("intrinsic {}", intrinsic),
 579             };
 580             let res = crate::num::codegen_int_binop(fx, bin_op, x, y);
 581             ret.write_cvalue(fx, res);
 582         };
 583         _ if intrinsic.ends_with("_with_overflow"), (c x, c y) {
 584             assert_eq!(x.layout().ty, y.layout().ty);
 585             let bin_op = match intrinsic {
 586                 "add_with_overflow" => BinOp::Add,
 587                 "sub_with_overflow" => BinOp::Sub,
 588                 "mul_with_overflow" => BinOp::Mul,
 589                 _ => unreachable!("intrinsic {}", intrinsic),
 590             };
 591
 592             let res = crate::num::codegen_checked_int_binop(
 593                 fx,
 594                 bin_op,
 595                 x,
 596                 y,
 597             );
 598             ret.write_cvalue(fx, res);
 599         };
 600         _ if intrinsic.starts_with("wrapping_"), (c x, c y) {
 601             assert_eq!(x.layout().ty, y.layout().ty);
 602             let bin_op = match intrinsic {
 603                 "wrapping_add" => BinOp::Add,
 604                 "wrapping_sub" => BinOp::Sub,
 605                 "wrapping_mul" => BinOp::Mul,
 606                 _ => unreachable!("intrinsic {}", intrinsic),
 607             };
 608             let res = crate::num::codegen_int_binop(
 609                 fx,
 610                 bin_op,
 611                 x,
 612                 y,
 613             );
 614             ret.write_cvalue(fx, res);
 615         };
 616         _ if intrinsic.starts_with("saturating_"), <T> (c lhs, c rhs) {
 617             assert_eq!(lhs.layout().ty, rhs.layout().ty);
 618             let bin_op = match intrinsic {
 619                 "saturating_add" => BinOp::Add,
 620                 "saturating_sub" => BinOp::Sub,
 621                 _ => unreachable!("intrinsic {}", intrinsic),
 622             };
 623
 624             let signed = type_sign(T);
 625
 626             let checked_res = crate::num::codegen_checked_int_binop(
 627                 fx,
 628                 bin_op,
 629                 lhs,
 630                 rhs,
 631             );
 632
 633             let (val, has_overflow) = checked_res.load_scalar_pair(fx);
 634             let clif_ty = fx.clif_type(T).unwrap();
 635
 636             // `select.i8` is not implemented by Cranelift.
 637             let has_overflow = fx.bcx.ins().uextend(types::I32, has_overflow);
 638
 639             let (min, max) = type_min_max_value(&mut fx.bcx, clif_ty, signed);
 640
 641             let val = match (intrinsic, signed) {
 642                 ("saturating_add", false) => fx.bcx.ins().select(has_overflow, max, val),
 643                 ("saturating_sub", false) => fx.bcx.ins().select(has_overflow, min, val),
 644                 ("saturating_add", true) => {
 645                     let rhs = rhs.load_scalar(fx);
 646                     let rhs_ge_zero = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0);
 647                     let sat_val = fx.bcx.ins().select(rhs_ge_zero, max, min);
 648                     fx.bcx.ins().select(has_overflow, sat_val, val)
 649                 }
 650                 ("saturating_sub", true) => {
 651                     let rhs = rhs.load_scalar(fx);
 652                     let rhs_ge_zero = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0);
 653                     let sat_val = fx.bcx.ins().select(rhs_ge_zero, min, max);
 654                     fx.bcx.ins().select(has_overflow, sat_val, val)
 655                 }
 656                 _ => unreachable!(),
 657             };
 658
 659             let res = CValue::by_val(val, fx.layout_of(T));
 660
 661             ret.write_cvalue(fx, res);
 662         };
 663         rotate_left, <T>(v x, v y) {
 664             let layout = fx.layout_of(T);
 665             let res = fx.bcx.ins().rotl(x, y);
 666             ret.write_cvalue(fx, CValue::by_val(res, layout));
 667         };
 668         rotate_right, <T>(v x, v y) {
 669             let layout = fx.layout_of(T);
 670             let res = fx.bcx.ins().rotr(x, y);
 671             ret.write_cvalue(fx, CValue::by_val(res, layout));
 672         };
 673
 674         // The only difference between offset and arith_offset is regarding UB. Because Cranelift
 675         // doesn't have UB both are codegen'ed the same way
 676         offset | arith_offset, (c base, v offset) {
 677             let pointee_ty = base.layout().ty.builtin_deref(true).unwrap().ty;
 678             let pointee_size = fx.layout_of(pointee_ty).size.bytes();
 679             let ptr_diff = fx.bcx.ins().imul_imm(offset, pointee_size as i64);
 680             let base_val = base.load_scalar(fx);
 681             let res = fx.bcx.ins().iadd(base_val, ptr_diff);
 682             ret.write_cvalue(fx, CValue::by_val(res, base.layout()));
 683         };
 684
 685         transmute, (c from) {
 686             ret.write_cvalue_transmute(fx, from);
 687         };
 688         write_bytes | volatile_set_memory, (c dst, v val, v count) {
 689             let pointee_ty = dst.layout().ty.builtin_deref(true).unwrap().ty;
 690             let pointee_size = fx.layout_of(pointee_ty).size.bytes();
 691             let count = fx.bcx.ins().imul_imm(count, pointee_size as i64);
 692             let dst_ptr = dst.load_scalar(fx);
 693             // FIXME make the memset actually volatile when switching to emit_small_memset
 694             // FIXME use emit_small_memset
 695             fx.bcx.call_memset(fx.cx.module.target_config(), dst_ptr, val, count);
 696         };
 697         ctlz | ctlz_nonzero, <T> (v arg) {
 698             // FIXME trap on `ctlz_nonzero` with zero arg.
 699             let res = if T == fx.tcx.types.u128 || T == fx.tcx.types.i128 {
 700                 // FIXME verify this algorithm is correct
 701                 let (lsb, msb) = fx.bcx.ins().isplit(arg);
 702                 let lsb_lz = fx.bcx.ins().clz(lsb);
 703                 let msb_lz = fx.bcx.ins().clz(msb);
 704                 let msb_is_zero = fx.bcx.ins().icmp_imm(IntCC::Equal, msb, 0);
 705                 let lsb_lz_plus_64 = fx.bcx.ins().iadd_imm(lsb_lz, 64);
 706                 let res = fx.bcx.ins().select(msb_is_zero, lsb_lz_plus_64, msb_lz);
 707                 fx.bcx.ins().uextend(types::I128, res)
 708             } else {
 709                 fx.bcx.ins().clz(arg)
 710             };
 711             let res = CValue::by_val(res, fx.layout_of(T));
 712             ret.write_cvalue(fx, res);
 713         };
 714         cttz | cttz_nonzero, <T> (v arg) {
 715             // FIXME trap on `cttz_nonzero` with zero arg.
 716             let res = if T == fx.tcx.types.u128 || T == fx.tcx.types.i128 {
 717                 // FIXME verify this algorithm is correct
 718                 let (lsb, msb) = fx.bcx.ins().isplit(arg);
 719                 let lsb_tz = fx.bcx.ins().ctz(lsb);
 720                 let msb_tz = fx.bcx.ins().ctz(msb);
 721                 let lsb_is_zero = fx.bcx.ins().icmp_imm(IntCC::Equal, lsb, 0);
 722                 let msb_tz_plus_64 = fx.bcx.ins().iadd_imm(msb_tz, 64);
 723                 let res = fx.bcx.ins().select(lsb_is_zero, msb_tz_plus_64, lsb_tz);
 724                 fx.bcx.ins().uextend(types::I128, res)
 725             } else {
 726                 fx.bcx.ins().ctz(arg)
 727             };
 728             let res = CValue::by_val(res, fx.layout_of(T));
 729             ret.write_cvalue(fx, res);
 730         };
 731         ctpop, <T> (v arg) {
 732             let res = fx.bcx.ins().popcnt(arg);
 733             let res = CValue::by_val(res, fx.layout_of(T));
 734             ret.write_cvalue(fx, res);
 735         };
 736         bitreverse, <T> (v arg) {
 737             let res = fx.bcx.ins().bitrev(arg);
 738             let res = CValue::by_val(res, fx.layout_of(T));
 739             ret.write_cvalue(fx, res);
 740         };
 741         bswap, <T> (v arg) {
 742             // FIXME(CraneStation/cranelift#794) add bswap instruction to cranelift
 743             fn swap(bcx: &mut FunctionBuilder<'_>, v: Value) -> Value {
 744                 match bcx.func.dfg.value_type(v) {
 745                     types::I8 => v,
 746
 747                     // https://code.woboq.org/gcc/include/bits/byteswap.h.html
 748                     types::I16 => {
 749                         let tmp1 = bcx.ins().ishl_imm(v, 8);
 750                         let n1 = bcx.ins().band_imm(tmp1, 0xFF00);
 751
 752                         let tmp2 = bcx.ins().ushr_imm(v, 8);
 753                         let n2 = bcx.ins().band_imm(tmp2, 0x00FF);
 754
 755                         bcx.ins().bor(n1, n2)
 756                     }
 757                     types::I32 => {
 758                         let tmp1 = bcx.ins().ishl_imm(v, 24);
 759                         let n1 = bcx.ins().band_imm(tmp1, 0xFF00_0000);
 760
 761                         let tmp2 = bcx.ins().ishl_imm(v, 8);
 762                         let n2 = bcx.ins().band_imm(tmp2, 0x00FF_0000);
 763
 764                         let tmp3 = bcx.ins().ushr_imm(v, 8);
 765                         let n3 = bcx.ins().band_imm(tmp3, 0x0000_FF00);
 766
 767                         let tmp4 = bcx.ins().ushr_imm(v, 24);
 768                         let n4 = bcx.ins().band_imm(tmp4, 0x0000_00FF);
 769
 770                         let or_tmp1 = bcx.ins().bor(n1, n2);
 771                         let or_tmp2 = bcx.ins().bor(n3, n4);
 772                         bcx.ins().bor(or_tmp1, or_tmp2)
 773                     }
 774                     types::I64 => {
 775                         let tmp1 = bcx.ins().ishl_imm(v, 56);
 776                         let n1 = bcx.ins().band_imm(tmp1, 0xFF00_0000_0000_0000u64 as i64);
 777
 778                         let tmp2 = bcx.ins().ishl_imm(v, 40);
 779                         let n2 = bcx.ins().band_imm(tmp2, 0x00FF_0000_0000_0000u64 as i64);
 780
 781                         let tmp3 = bcx.ins().ishl_imm(v, 24);
 782                         let n3 = bcx.ins().band_imm(tmp3, 0x0000_FF00_0000_0000u64 as i64);
 783
 784                         let tmp4 = bcx.ins().ishl_imm(v, 8);
 785                         let n4 = bcx.ins().band_imm(tmp4, 0x0000_00FF_0000_0000u64 as i64);
 786
 787                         let tmp5 = bcx.ins().ushr_imm(v, 8);
 788                         let n5 = bcx.ins().band_imm(tmp5, 0x0000_0000_FF00_0000u64 as i64);
 789
 790                         let tmp6 = bcx.ins().ushr_imm(v, 24);
 791                         let n6 = bcx.ins().band_imm(tmp6, 0x0000_0000_00FF_0000u64 as i64);
 792
 793                         let tmp7 = bcx.ins().ushr_imm(v, 40);
 794                         let n7 = bcx.ins().band_imm(tmp7, 0x0000_0000_0000_FF00u64 as i64);
 795
 796                         let tmp8 = bcx.ins().ushr_imm(v, 56);
 797                         let n8 = bcx.ins().band_imm(tmp8, 0x0000_0000_0000_00FFu64 as i64);
 798
 799                         let or_tmp1 = bcx.ins().bor(n1, n2);
 800                         let or_tmp2 = bcx.ins().bor(n3, n4);
 801                         let or_tmp3 = bcx.ins().bor(n5, n6);
 802                         let or_tmp4 = bcx.ins().bor(n7, n8);
 803
 804                         let or_tmp5 = bcx.ins().bor(or_tmp1, or_tmp2);
 805                         let or_tmp6 = bcx.ins().bor(or_tmp3, or_tmp4);
 806                         bcx.ins().bor(or_tmp5, or_tmp6)
 807                     }
 808                     types::I128 => {
 809                         let (lo, hi) = bcx.ins().isplit(v);
 810                         let lo = swap(bcx, lo);
 811                         let hi = swap(bcx, hi);
 812                         bcx.ins().iconcat(hi, lo)
 813                     }
 814                     ty => unreachable!("bswap {}", ty),
 815                 }
 816             };
 817             let res = CValue::by_val(swap(&mut fx.bcx, arg), fx.layout_of(T));
 818             ret.write_cvalue(fx, res);
 819         };
 820         assert_inhabited | assert_zero_valid | assert_uninit_valid, <T> () {
 821             let layout = fx.layout_of(T);
 822             if layout.abi.is_uninhabited() {
 823                 with_no_trimmed_paths(|| crate::base::codegen_panic(
 824                     fx,
 825                     &format!("attempted to instantiate uninhabited type `{}`", T),
 826                     span,
 827                 ));
 828                 return;
 829             }
 830
 831             if intrinsic == "assert_zero_valid" && !layout.might_permit_raw_init(fx, /*zero:*/ true).unwrap() {
 832                 with_no_trimmed_paths(|| crate::base::codegen_panic(
 833                     fx,
 834                     &format!("attempted to zero-initialize type `{}`, which is invalid", T),
 835                     span,
 836                 ));
 837                 return;
 838             }
 839
 840             if intrinsic == "assert_uninit_valid" && !layout.might_permit_raw_init(fx, /*zero:*/ false).unwrap() {
 841                 with_no_trimmed_paths(|| crate::base::codegen_panic(
 842                     fx,
 843                     &format!("attempted to leave type `{}` uninitialized, which is invalid", T),
 844                     span,
 845                 ));
 846                 return;
 847             }
 848         };
 849
 850         volatile_load | unaligned_volatile_load, (c ptr) {
 851             // Cranelift treats loads as volatile by default
 852             // FIXME ignore during stack2reg optimization
 853             // FIXME correctly handle unaligned_volatile_load
 854             let inner_layout =
 855                 fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty);
 856             let val = CValue::by_ref(Pointer::new(ptr.load_scalar(fx)), inner_layout);
 857             ret.write_cvalue(fx, val);
 858         };
 859         volatile_store | unaligned_volatile_store, (v ptr, c val) {
 860             // Cranelift treats stores as volatile by default
 861             // FIXME ignore during stack2reg optimization
 862             // FIXME correctly handle unaligned_volatile_store
 863             let dest = CPlace::for_ptr(Pointer::new(ptr), val.layout());
 864             dest.write_cvalue(fx, val);
 865         };
 866
 867         size_of | pref_align_of | min_align_of | needs_drop | type_id | type_name | variant_count, () {
 868             let const_val =
 869                 fx.tcx.const_eval_instance(ParamEnv::reveal_all(), instance, None).unwrap();
 870             let val = crate::constant::codegen_const_value(
 871                 fx,
 872                 const_val,
 873                 ret.layout().ty,
 874             );
 875             ret.write_cvalue(fx, val);
 876         };
 877
 878         ptr_offset_from, <T> (v ptr, v base) {
 879             let isize_layout = fx.layout_of(fx.tcx.types.isize);
 880
 881             let pointee_size: u64 = fx.layout_of(T).size.bytes();
 882             let diff = fx.bcx.ins().isub(ptr, base);
 883             // FIXME this can be an exact division.
 884             let val = CValue::by_val(fx.bcx.ins().sdiv_imm(diff, pointee_size as i64), isize_layout);
 885             ret.write_cvalue(fx, val);
 886         };
 887
 888         ptr_guaranteed_eq, (c a, c b) {
 889             let val = crate::num::codegen_ptr_binop(fx, BinOp::Eq, a, b);
 890             ret.write_cvalue(fx, val);
 891         };
 892
 893         ptr_guaranteed_ne, (c a, c b) {
 894             let val = crate::num::codegen_ptr_binop(fx, BinOp::Ne, a, b);
 895             ret.write_cvalue(fx, val);
 896         };
 897
 898         caller_location, () {
 899             let caller_location = fx.get_caller_location(span);
 900             ret.write_cvalue(fx, caller_location);
 901         };
 902
 903         _ if intrinsic.starts_with("atomic_fence"), () {
 904             crate::atomic_shim::lock_global_lock(fx);
 905             crate::atomic_shim::unlock_global_lock(fx);
 906         };
 907         _ if intrinsic.starts_with("atomic_singlethreadfence"), () {
 908             crate::atomic_shim::lock_global_lock(fx);
 909             crate::atomic_shim::unlock_global_lock(fx);
 910         };
 911         _ if intrinsic.starts_with("atomic_load"), (c ptr) {
 912             crate::atomic_shim::lock_global_lock(fx);
 913
 914             let inner_layout =
 915                 fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty);
 916             validate_atomic_type!(fx, intrinsic, span, inner_layout.ty);
 917             let val = CValue::by_ref(Pointer::new(ptr.load_scalar(fx)), inner_layout);
 918             ret.write_cvalue(fx, val);
 919
 920             crate::atomic_shim::unlock_global_lock(fx);
 921         };
 922         _ if intrinsic.starts_with("atomic_store"), (v ptr, c val) {
 923             validate_atomic_type!(fx, intrinsic, span, val.layout().ty);
 924
 925             crate::atomic_shim::lock_global_lock(fx);
 926
 927             let dest = CPlace::for_ptr(Pointer::new(ptr), val.layout());
 928             dest.write_cvalue(fx, val);
 929
 930             crate::atomic_shim::unlock_global_lock(fx);
 931         };
 932         _ if intrinsic.starts_with("atomic_xchg"), <T> (v ptr, c src) {
 933             validate_atomic_type!(fx, intrinsic, span, T);
 934
 935             crate::atomic_shim::lock_global_lock(fx);
 936
 937             // Read old
 938             let clif_ty = fx.clif_type(T).unwrap();
 939             let old = fx.bcx.ins().load(clif_ty, MemFlags::new(), ptr, 0);
 940             ret.write_cvalue(fx, CValue::by_val(old, fx.layout_of(T)));
 941
 942             // Write new
 943             let dest = CPlace::for_ptr(Pointer::new(ptr), src.layout());
 944             dest.write_cvalue(fx, src);
 945
 946             crate::atomic_shim::unlock_global_lock(fx);
 947         };
 948         _ if intrinsic.starts_with("atomic_cxchg"), <T> (v ptr, c test_old, c new) { // both atomic_cxchg_* and atomic_cxchgweak_*
 949             validate_atomic_type!(fx, intrinsic, span, T);
 950
 951             let test_old = test_old.load_scalar(fx);
 952             let new = new.load_scalar(fx);
 953
 954             crate::atomic_shim::lock_global_lock(fx);
 955
 956             // Read old
 957             let clif_ty = fx.clif_type(T).unwrap();
 958             let old = fx.bcx.ins().load(clif_ty, MemFlags::new(), ptr, 0);
 959
 960             // Compare
 961             let is_eq = fx.bcx.ins().icmp(IntCC::Equal, old, test_old);
 962             let new = fx.bcx.ins().select(is_eq, new, old); // Keep old if not equal to test_old
 963
 964             // Write new
 965             fx.bcx.ins().store(MemFlags::new(), new, ptr, 0);
 966
 967             let ret_val = CValue::by_val_pair(old, fx.bcx.ins().bint(types::I8, is_eq), ret.layout());
 968             ret.write_cvalue(fx, ret_val);
 969
 970             crate::atomic_shim::unlock_global_lock(fx);
 971         };
 972
 973         _ if intrinsic.starts_with("atomic_xadd"), <T> (v ptr, c amount) {
 974             validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
 975             let amount = amount.load_scalar(fx);
 976             atomic_binop_return_old! (fx, iadd<T>(ptr, amount) -> ret);
 977         };
 978         _ if intrinsic.starts_with("atomic_xsub"), <T> (v ptr, c amount) {
 979             validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
 980             let amount = amount.load_scalar(fx);
 981             atomic_binop_return_old! (fx, isub<T>(ptr, amount) -> ret);
 982         };
 983         _ if intrinsic.starts_with("atomic_and"), <T> (v ptr, c src) {
 984             validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
 985             let src = src.load_scalar(fx);
 986             atomic_binop_return_old! (fx, band<T>(ptr, src) -> ret);
 987         };
 988         _ if intrinsic.starts_with("atomic_nand"), <T> (v ptr, c src) {
 989             validate_atomic_type!(fx, intrinsic, span, T);
 990
 991             let src = src.load_scalar(fx);
 992
 993             crate::atomic_shim::lock_global_lock(fx);
 994
 995             let clif_ty = fx.clif_type(T).unwrap();
 996             let old = fx.bcx.ins().load(clif_ty, MemFlags::new(), ptr, 0);
 997             let and = fx.bcx.ins().band(old, src);
 998             let new = fx.bcx.ins().bnot(and);
 999             fx.bcx.ins().store(MemFlags::new(), new, ptr, 0);
1000             ret.write_cvalue(fx, CValue::by_val(old, fx.layout_of(T)));
1001
1002             crate::atomic_shim::unlock_global_lock(fx);
1003         };
1004         _ if intrinsic.starts_with("atomic_or"), <T> (v ptr, c src) {
1005             validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
1006             let src = src.load_scalar(fx);
1007             atomic_binop_return_old! (fx, bor<T>(ptr, src) -> ret);
1008         };
1009         _ if intrinsic.starts_with("atomic_xor"), <T> (v ptr, c src) {
1010             validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
1011             let src = src.load_scalar(fx);
1012             atomic_binop_return_old! (fx, bxor<T>(ptr, src) -> ret);
1013         };
1014
1015         _ if intrinsic.starts_with("atomic_max"), <T> (v ptr, c src) {
1016             validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
1017             let src = src.load_scalar(fx);
1018             atomic_minmax!(fx, IntCC::SignedGreaterThan, <T> (ptr, src) -> ret);
1019         };
1020         _ if intrinsic.starts_with("atomic_umax"), <T> (v ptr, c src) {
1021             validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
1022             let src = src.load_scalar(fx);
1023             atomic_minmax!(fx, IntCC::UnsignedGreaterThan, <T> (ptr, src) -> ret);
1024         };
1025         _ if intrinsic.starts_with("atomic_min"), <T> (v ptr, c src) {
1026             validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
1027             let src = src.load_scalar(fx);
1028             atomic_minmax!(fx, IntCC::SignedLessThan, <T> (ptr, src) -> ret);
1029         };
1030         _ if intrinsic.starts_with("atomic_umin"), <T> (v ptr, c src) {
1031             validate_atomic_type!(fx, intrinsic, span, ret.layout().ty);
1032             let src = src.load_scalar(fx);
1033             atomic_minmax!(fx, IntCC::UnsignedLessThan, <T> (ptr, src) -> ret);
1034         };
1035
1036         minnumf32, (v a, v b) {
1037             let val = fx.bcx.ins().fmin(a, b);
1038             let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f32));
1039             ret.write_cvalue(fx, val);
1040         };
1041         minnumf64, (v a, v b) {
1042             let val = fx.bcx.ins().fmin(a, b);
1043             let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f64));
1044             ret.write_cvalue(fx, val);
1045         };
1046         maxnumf32, (v a, v b) {
1047             let val = fx.bcx.ins().fmax(a, b);
1048             let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f32));
1049             ret.write_cvalue(fx, val);
1050         };
1051         maxnumf64, (v a, v b) {
1052             let val = fx.bcx.ins().fmax(a, b);
1053             let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f64));
1054             ret.write_cvalue(fx, val);
1055         };
1056
1057         try, (v f, v data, v _catch_fn) {
1058             // FIXME once unwinding is supported, change this to actually catch panics
1059             let f_sig = fx.bcx.func.import_signature(Signature {
1060                 call_conv: CallConv::triple_default(fx.triple()),
1061                 params: vec![AbiParam::new(fx.bcx.func.dfg.value_type(data))],
1062                 returns: vec![],
1063             });
1064
1065             fx.bcx.ins().call_indirect(f_sig, f, &[data]);
1066
1067             let layout = ret.layout();
1068             let ret_val = CValue::const_val(fx, layout, ty::ScalarInt::null(layout.size));
1069             ret.write_cvalue(fx, ret_val);
1070         };
1071
1072         fadd_fast | fsub_fast | fmul_fast | fdiv_fast | frem_fast, (c x, c y) {
1073             let res = crate::num::codegen_float_binop(fx, match intrinsic {
1074                 "fadd_fast" => BinOp::Add,
1075                 "fsub_fast" => BinOp::Sub,
1076                 "fmul_fast" => BinOp::Mul,
1077                 "fdiv_fast" => BinOp::Div,
1078                 "frem_fast" => BinOp::Rem,
1079                 _ => unreachable!(),
1080             }, x, y);
1081             ret.write_cvalue(fx, res);
1082         };
1083         float_to_int_unchecked, (v f) {
1084             let res = crate::cast::clif_int_or_float_cast(
1085                 fx,
1086                 f,
1087                 false,
1088                 fx.clif_type(ret.layout().ty).unwrap(),
1089                 type_sign(ret.layout().ty),
1090             );
1091             ret.write_cvalue(fx, CValue::by_val(res, ret.layout()));
1092         };
1093     }
1094
1095     if let Some((_, dest)) = destination {
1096         let ret_block = fx.get_block(dest);
1097         fx.bcx.ins().jump(ret_block, &[]);
1098     } else {
1099         trap_unreachable(fx, "[corruption] Diverging intrinsic returned.");
1100     }
1101 }