compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs

   1 //! Codegen of intrinsics. This includes `extern "rust-intrinsic"`, `extern "platform-intrinsic"`
   2 //! and LLVM intrinsics that have symbol names starting with `llvm.`.
   3
   4 macro_rules! intrinsic_args {
   5     ($fx:expr, $args:expr => ($($arg:tt),*); $intrinsic:expr) => {
   6         #[allow(unused_parens)]
   7         let ($($arg),*) = if let [$($arg),*] = $args {
   8             ($(codegen_operand($fx, $arg)),*)
   9         } else {
  10             $crate::intrinsics::bug_on_incorrect_arg_count($intrinsic);
  11         };
  12     }
  13 }
  14
  15 mod cpuid;
  16 mod llvm;
  17 mod llvm_aarch64;
  18 mod llvm_x86;
  19 mod simd;
  20
  21 pub(crate) use cpuid::codegen_cpuid_call;
  22 pub(crate) use llvm::codegen_llvm_intrinsic_call;
  23
  24 use rustc_middle::ty;
  25 use rustc_middle::ty::layout::{HasParamEnv, ValidityRequirement};
  26 use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths};
  27 use rustc_middle::ty::GenericArgsRef;
  28 use rustc_span::symbol::{kw, sym, Symbol};
  29
  30 use crate::prelude::*;
  31 use cranelift_codegen::ir::AtomicRmwOp;
  32
  33 fn bug_on_incorrect_arg_count(intrinsic: impl std::fmt::Display) -> ! {
  34     bug!("wrong number of args for intrinsic {}", intrinsic);
  35 }
  36
  37 fn report_atomic_type_validation_error<'tcx>(
  38     fx: &mut FunctionCx<'_, '_, 'tcx>,
  39     intrinsic: Symbol,
  40     span: Span,
  41     ty: Ty<'tcx>,
  42 ) {
  43     fx.tcx.sess.span_err(
  44         span,
  45         format!(
  46             "`{}` intrinsic: expected basic integer or raw pointer type, found `{:?}`",
  47             intrinsic, ty
  48         ),
  49     );
  50     // Prevent verifier error
  51     fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
  52 }
  53
  54 pub(crate) fn clif_vector_type<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx>) -> Type {
  55     let (element, count) = match layout.abi {
  56         Abi::Vector { element, count } => (element, count),
  57         _ => unreachable!(),
  58     };
  59
  60     scalar_to_clif_type(tcx, element).by(u32::try_from(count).unwrap()).unwrap()
  61 }
  62
  63 fn simd_for_each_lane<'tcx>(
  64     fx: &mut FunctionCx<'_, '_, 'tcx>,
  65     val: CValue<'tcx>,
  66     ret: CPlace<'tcx>,
  67     f: &dyn Fn(&mut FunctionCx<'_, '_, 'tcx>, Ty<'tcx>, Ty<'tcx>, Value) -> Value,
  68 ) {
  69     let layout = val.layout();
  70
  71     let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
  72     let lane_layout = fx.layout_of(lane_ty);
  73     let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
  74     let ret_lane_layout = fx.layout_of(ret_lane_ty);
  75     assert_eq!(lane_count, ret_lane_count);
  76
  77     for lane_idx in 0..lane_count {
  78         let lane = val.value_lane(fx, lane_idx).load_scalar(fx);
  79
  80         let res_lane = f(fx, lane_layout.ty, ret_lane_layout.ty, lane);
  81         let res_lane = CValue::by_val(res_lane, ret_lane_layout);
  82
  83         ret.place_lane(fx, lane_idx).write_cvalue(fx, res_lane);
  84     }
  85 }
  86
  87 fn simd_pair_for_each_lane_typed<'tcx>(
  88     fx: &mut FunctionCx<'_, '_, 'tcx>,
  89     x: CValue<'tcx>,
  90     y: CValue<'tcx>,
  91     ret: CPlace<'tcx>,
  92     f: &dyn Fn(&mut FunctionCx<'_, '_, 'tcx>, CValue<'tcx>, CValue<'tcx>) -> CValue<'tcx>,
  93 ) {
  94     assert_eq!(x.layout(), y.layout());
  95     let layout = x.layout();
  96
  97     let (lane_count, _lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
  98     let (ret_lane_count, _ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
  99     assert_eq!(lane_count, ret_lane_count);
 100
 101     for lane_idx in 0..lane_count {
 102         let x_lane = x.value_lane(fx, lane_idx);
 103         let y_lane = y.value_lane(fx, lane_idx);
 104
 105         let res_lane = f(fx, x_lane, y_lane);
 106
 107         ret.place_lane(fx, lane_idx).write_cvalue(fx, res_lane);
 108     }
 109 }
 110
 111 fn simd_pair_for_each_lane<'tcx>(
 112     fx: &mut FunctionCx<'_, '_, 'tcx>,
 113     x: CValue<'tcx>,
 114     y: CValue<'tcx>,
 115     ret: CPlace<'tcx>,
 116     f: &dyn Fn(&mut FunctionCx<'_, '_, 'tcx>, Ty<'tcx>, Ty<'tcx>, Value, Value) -> Value,
 117 ) {
 118     assert_eq!(x.layout(), y.layout());
 119     let layout = x.layout();
 120
 121     let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
 122     let lane_layout = fx.layout_of(lane_ty);
 123     let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
 124     let ret_lane_layout = fx.layout_of(ret_lane_ty);
 125     assert_eq!(lane_count, ret_lane_count);
 126
 127     for lane_idx in 0..lane_count {
 128         let x_lane = x.value_lane(fx, lane_idx).load_scalar(fx);
 129         let y_lane = y.value_lane(fx, lane_idx).load_scalar(fx);
 130
 131         let res_lane = f(fx, lane_layout.ty, ret_lane_layout.ty, x_lane, y_lane);
 132         let res_lane = CValue::by_val(res_lane, ret_lane_layout);
 133
 134         ret.place_lane(fx, lane_idx).write_cvalue(fx, res_lane);
 135     }
 136 }
 137
 138 fn simd_reduce<'tcx>(
 139     fx: &mut FunctionCx<'_, '_, 'tcx>,
 140     val: CValue<'tcx>,
 141     acc: Option<Value>,
 142     ret: CPlace<'tcx>,
 143     f: &dyn Fn(&mut FunctionCx<'_, '_, 'tcx>, Ty<'tcx>, Value, Value) -> Value,
 144 ) {
 145     let (lane_count, lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx);
 146     let lane_layout = fx.layout_of(lane_ty);
 147     assert_eq!(lane_layout, ret.layout());
 148
 149     let (mut res_val, start_lane) =
 150         if let Some(acc) = acc { (acc, 0) } else { (val.value_lane(fx, 0).load_scalar(fx), 1) };
 151     for lane_idx in start_lane..lane_count {
 152         let lane = val.value_lane(fx, lane_idx).load_scalar(fx);
 153         res_val = f(fx, lane_layout.ty, res_val, lane);
 154     }
 155     let res = CValue::by_val(res_val, lane_layout);
 156     ret.write_cvalue(fx, res);
 157 }
 158
 159 // FIXME move all uses to `simd_reduce`
 160 fn simd_reduce_bool<'tcx>(
 161     fx: &mut FunctionCx<'_, '_, 'tcx>,
 162     val: CValue<'tcx>,
 163     ret: CPlace<'tcx>,
 164     f: &dyn Fn(&mut FunctionCx<'_, '_, 'tcx>, Value, Value) -> Value,
 165 ) {
 166     let (lane_count, _lane_ty) = val.layout().ty.simd_size_and_type(fx.tcx);
 167     assert!(ret.layout().ty.is_bool());
 168
 169     let res_val = val.value_lane(fx, 0).load_scalar(fx);
 170     let mut res_val = fx.bcx.ins().band_imm(res_val, 1); // mask to boolean
 171     for lane_idx in 1..lane_count {
 172         let lane = val.value_lane(fx, lane_idx).load_scalar(fx);
 173         let lane = fx.bcx.ins().band_imm(lane, 1); // mask to boolean
 174         res_val = f(fx, res_val, lane);
 175     }
 176     let res_val = if fx.bcx.func.dfg.value_type(res_val) != types::I8 {
 177         fx.bcx.ins().ireduce(types::I8, res_val)
 178     } else {
 179         res_val
 180     };
 181     let res = CValue::by_val(res_val, ret.layout());
 182     ret.write_cvalue(fx, res);
 183 }
 184
 185 fn bool_to_zero_or_max_uint<'tcx>(
 186     fx: &mut FunctionCx<'_, '_, 'tcx>,
 187     ty: Ty<'tcx>,
 188     val: Value,
 189 ) -> Value {
 190     let ty = fx.clif_type(ty).unwrap();
 191
 192     let int_ty = match ty {
 193         types::F32 => types::I32,
 194         types::F64 => types::I64,
 195         ty => ty,
 196     };
 197
 198     let mut res = fx.bcx.ins().bmask(int_ty, val);
 199
 200     if ty.is_float() {
 201         res = codegen_bitcast(fx, ty, res);
 202     }
 203
 204     res
 205 }
 206
 207 pub(crate) fn codegen_intrinsic_call<'tcx>(
 208     fx: &mut FunctionCx<'_, '_, 'tcx>,
 209     instance: Instance<'tcx>,
 210     args: &[mir::Operand<'tcx>],
 211     destination: CPlace<'tcx>,
 212     target: Option<BasicBlock>,
 213     source_info: mir::SourceInfo,
 214 ) {
 215     let intrinsic = fx.tcx.item_name(instance.def_id());
 216     let instance_args = instance.args;
 217
 218     if intrinsic.as_str().starts_with("simd_") {
 219         self::simd::codegen_simd_intrinsic_call(
 220             fx,
 221             intrinsic,
 222             instance_args,
 223             args,
 224             destination,
 225             target.expect("target for simd intrinsic"),
 226             source_info.span,
 227         );
 228     } else if codegen_float_intrinsic_call(fx, intrinsic, args, destination) {
 229         let ret_block = fx.get_block(target.expect("target for float intrinsic"));
 230         fx.bcx.ins().jump(ret_block, &[]);
 231     } else {
 232         codegen_regular_intrinsic_call(
 233             fx,
 234             instance,
 235             intrinsic,
 236             instance_args,
 237             args,
 238             destination,
 239             target,
 240             source_info,
 241         );
 242     }
 243 }
 244
 245 fn codegen_float_intrinsic_call<'tcx>(
 246     fx: &mut FunctionCx<'_, '_, 'tcx>,
 247     intrinsic: Symbol,
 248     args: &[mir::Operand<'tcx>],
 249     ret: CPlace<'tcx>,
 250 ) -> bool {
 251     let (name, arg_count, ty, clif_ty) = match intrinsic {
 252         sym::expf32 => ("expf", 1, fx.tcx.types.f32, types::F32),
 253         sym::expf64 => ("exp", 1, fx.tcx.types.f64, types::F64),
 254         sym::exp2f32 => ("exp2f", 1, fx.tcx.types.f32, types::F32),
 255         sym::exp2f64 => ("exp2", 1, fx.tcx.types.f64, types::F64),
 256         sym::sqrtf32 => ("sqrtf", 1, fx.tcx.types.f32, types::F32),
 257         sym::sqrtf64 => ("sqrt", 1, fx.tcx.types.f64, types::F64),
 258         sym::powif32 => ("__powisf2", 2, fx.tcx.types.f32, types::F32), // compiler-builtins
 259         sym::powif64 => ("__powidf2", 2, fx.tcx.types.f64, types::F64), // compiler-builtins
 260         sym::powf32 => ("powf", 2, fx.tcx.types.f32, types::F32),
 261         sym::powf64 => ("pow", 2, fx.tcx.types.f64, types::F64),
 262         sym::logf32 => ("logf", 1, fx.tcx.types.f32, types::F32),
 263         sym::logf64 => ("log", 1, fx.tcx.types.f64, types::F64),
 264         sym::log2f32 => ("log2f", 1, fx.tcx.types.f32, types::F32),
 265         sym::log2f64 => ("log2", 1, fx.tcx.types.f64, types::F64),
 266         sym::log10f32 => ("log10f", 1, fx.tcx.types.f32, types::F32),
 267         sym::log10f64 => ("log10", 1, fx.tcx.types.f64, types::F64),
 268         sym::fabsf32 => ("fabsf", 1, fx.tcx.types.f32, types::F32),
 269         sym::fabsf64 => ("fabs", 1, fx.tcx.types.f64, types::F64),
 270         sym::fmaf32 => ("fmaf", 3, fx.tcx.types.f32, types::F32),
 271         sym::fmaf64 => ("fma", 3, fx.tcx.types.f64, types::F64),
 272         sym::copysignf32 => ("copysignf", 2, fx.tcx.types.f32, types::F32),
 273         sym::copysignf64 => ("copysign", 2, fx.tcx.types.f64, types::F64),
 274         sym::floorf32 => ("floorf", 1, fx.tcx.types.f32, types::F32),
 275         sym::floorf64 => ("floor", 1, fx.tcx.types.f64, types::F64),
 276         sym::ceilf32 => ("ceilf", 1, fx.tcx.types.f32, types::F32),
 277         sym::ceilf64 => ("ceil", 1, fx.tcx.types.f64, types::F64),
 278         sym::truncf32 => ("truncf", 1, fx.tcx.types.f32, types::F32),
 279         sym::truncf64 => ("trunc", 1, fx.tcx.types.f64, types::F64),
 280         sym::rintf32 => ("rintf", 1, fx.tcx.types.f32, types::F32),
 281         sym::rintf64 => ("rint", 1, fx.tcx.types.f64, types::F64),
 282         sym::roundf32 => ("roundf", 1, fx.tcx.types.f32, types::F32),
 283         sym::roundf64 => ("round", 1, fx.tcx.types.f64, types::F64),
 284         sym::roundevenf32 => ("roundevenf", 1, fx.tcx.types.f32, types::F32),
 285         sym::roundevenf64 => ("roundeven", 1, fx.tcx.types.f64, types::F64),
 286         sym::sinf32 => ("sinf", 1, fx.tcx.types.f32, types::F32),
 287         sym::sinf64 => ("sin", 1, fx.tcx.types.f64, types::F64),
 288         sym::cosf32 => ("cosf", 1, fx.tcx.types.f32, types::F32),
 289         sym::cosf64 => ("cos", 1, fx.tcx.types.f64, types::F64),
 290         _ => return false,
 291     };
 292
 293     if args.len() != arg_count {
 294         bug!("wrong number of args for intrinsic {:?}", intrinsic);
 295     }
 296
 297     let (a, b, c);
 298     let args = match args {
 299         [x] => {
 300             a = [codegen_operand(fx, x).load_scalar(fx)];
 301             &a as &[_]
 302         }
 303         [x, y] => {
 304             b = [codegen_operand(fx, x).load_scalar(fx), codegen_operand(fx, y).load_scalar(fx)];
 305             &b
 306         }
 307         [x, y, z] => {
 308             c = [
 309                 codegen_operand(fx, x).load_scalar(fx),
 310                 codegen_operand(fx, y).load_scalar(fx),
 311                 codegen_operand(fx, z).load_scalar(fx),
 312             ];
 313             &c
 314         }
 315         _ => unreachable!(),
 316     };
 317
 318     let layout = fx.layout_of(ty);
 319     let res = match intrinsic {
 320         sym::fmaf32 | sym::fmaf64 => {
 321             CValue::by_val(fx.bcx.ins().fma(args[0], args[1], args[2]), layout)
 322         }
 323         sym::copysignf32 | sym::copysignf64 => {
 324             CValue::by_val(fx.bcx.ins().fcopysign(args[0], args[1]), layout)
 325         }
 326         sym::fabsf32
 327         | sym::fabsf64
 328         | sym::floorf32
 329         | sym::floorf64
 330         | sym::ceilf32
 331         | sym::ceilf64
 332         | sym::truncf32
 333         | sym::truncf64 => {
 334             let val = match intrinsic {
 335                 sym::fabsf32 | sym::fabsf64 => fx.bcx.ins().fabs(args[0]),
 336                 sym::floorf32 | sym::floorf64 => fx.bcx.ins().floor(args[0]),
 337                 sym::ceilf32 | sym::ceilf64 => fx.bcx.ins().ceil(args[0]),
 338                 sym::truncf32 | sym::truncf64 => fx.bcx.ins().trunc(args[0]),
 339                 _ => unreachable!(),
 340             };
 341
 342             CValue::by_val(val, layout)
 343         }
 344
 345         // These intrinsics aren't supported natively by Cranelift.
 346         // Lower them to a libcall.
 347         sym::powif32 | sym::powif64 => {
 348             let input_tys: Vec<_> = vec![AbiParam::new(clif_ty), AbiParam::new(types::I32)];
 349             let ret_val = fx.lib_call(name, input_tys, vec![AbiParam::new(clif_ty)], &args)[0];
 350             CValue::by_val(ret_val, fx.layout_of(ty))
 351         }
 352         _ => {
 353             let input_tys: Vec<_> = args.iter().map(|_| AbiParam::new(clif_ty)).collect();
 354             let ret_val = fx.lib_call(name, input_tys, vec![AbiParam::new(clif_ty)], &args)[0];
 355             CValue::by_val(ret_val, fx.layout_of(ty))
 356         }
 357     };
 358
 359     ret.write_cvalue(fx, res);
 360
 361     true
 362 }
 363
 364 fn codegen_regular_intrinsic_call<'tcx>(
 365     fx: &mut FunctionCx<'_, '_, 'tcx>,
 366     instance: Instance<'tcx>,
 367     intrinsic: Symbol,
 368     generic_args: GenericArgsRef<'tcx>,
 369     args: &[mir::Operand<'tcx>],
 370     ret: CPlace<'tcx>,
 371     destination: Option<BasicBlock>,
 372     source_info: mir::SourceInfo,
 373 ) {
 374     let usize_layout = fx.layout_of(fx.tcx.types.usize);
 375
 376     match intrinsic {
 377         sym::abort => {
 378             fx.bcx.ins().trap(TrapCode::User(0));
 379             return;
 380         }
 381         sym::likely | sym::unlikely => {
 382             intrinsic_args!(fx, args => (a); intrinsic);
 383
 384             ret.write_cvalue(fx, a);
 385         }
 386         sym::breakpoint => {
 387             intrinsic_args!(fx, args => (); intrinsic);
 388
 389             fx.bcx.ins().debugtrap();
 390         }
 391         sym::copy => {
 392             intrinsic_args!(fx, args => (src, dst, count); intrinsic);
 393             let src = src.load_scalar(fx);
 394             let dst = dst.load_scalar(fx);
 395             let count = count.load_scalar(fx);
 396
 397             let elem_ty = generic_args.type_at(0);
 398             let elem_size: u64 = fx.layout_of(elem_ty).size.bytes();
 399             assert_eq!(args.len(), 3);
 400             let byte_amount =
 401                 if elem_size != 1 { fx.bcx.ins().imul_imm(count, elem_size as i64) } else { count };
 402
 403             // FIXME emit_small_memmove
 404             fx.bcx.call_memmove(fx.target_config, dst, src, byte_amount);
 405         }
 406         sym::volatile_copy_memory | sym::volatile_copy_nonoverlapping_memory => {
 407             // NOTE: the volatile variants have src and dst swapped
 408             intrinsic_args!(fx, args => (dst, src, count); intrinsic);
 409             let dst = dst.load_scalar(fx);
 410             let src = src.load_scalar(fx);
 411             let count = count.load_scalar(fx);
 412
 413             let elem_ty = generic_args.type_at(0);
 414             let elem_size: u64 = fx.layout_of(elem_ty).size.bytes();
 415             assert_eq!(args.len(), 3);
 416             let byte_amount =
 417                 if elem_size != 1 { fx.bcx.ins().imul_imm(count, elem_size as i64) } else { count };
 418
 419             // FIXME make the copy actually volatile when using emit_small_mem{cpy,move}
 420             if intrinsic == sym::volatile_copy_nonoverlapping_memory {
 421                 // FIXME emit_small_memcpy
 422                 fx.bcx.call_memcpy(fx.target_config, dst, src, byte_amount);
 423             } else {
 424                 // FIXME emit_small_memmove
 425                 fx.bcx.call_memmove(fx.target_config, dst, src, byte_amount);
 426             }
 427         }
 428         sym::size_of_val => {
 429             intrinsic_args!(fx, args => (ptr); intrinsic);
 430
 431             let layout = fx.layout_of(generic_args.type_at(0));
 432             // Note: Can't use is_unsized here as truly unsized types need to take the fixed size
 433             // branch
 434             let size = if let Abi::ScalarPair(_, _) = ptr.layout().abi {
 435                 let (_ptr, info) = ptr.load_scalar_pair(fx);
 436                 let (size, _align) = crate::unsize::size_and_align_of_dst(fx, layout, info);
 437                 size
 438             } else {
 439                 fx.bcx.ins().iconst(fx.pointer_type, layout.size.bytes() as i64)
 440             };
 441             ret.write_cvalue(fx, CValue::by_val(size, usize_layout));
 442         }
 443         sym::min_align_of_val => {
 444             intrinsic_args!(fx, args => (ptr); intrinsic);
 445
 446             let layout = fx.layout_of(generic_args.type_at(0));
 447             // Note: Can't use is_unsized here as truly unsized types need to take the fixed size
 448             // branch
 449             let align = if let Abi::ScalarPair(_, _) = ptr.layout().abi {
 450                 let (_ptr, info) = ptr.load_scalar_pair(fx);
 451                 let (_size, align) = crate::unsize::size_and_align_of_dst(fx, layout, info);
 452                 align
 453             } else {
 454                 fx.bcx.ins().iconst(fx.pointer_type, layout.align.abi.bytes() as i64)
 455             };
 456             ret.write_cvalue(fx, CValue::by_val(align, usize_layout));
 457         }
 458
 459         sym::vtable_size => {
 460             intrinsic_args!(fx, args => (vtable); intrinsic);
 461             let vtable = vtable.load_scalar(fx);
 462
 463             let size = crate::vtable::size_of_obj(fx, vtable);
 464             ret.write_cvalue(fx, CValue::by_val(size, usize_layout));
 465         }
 466
 467         sym::vtable_align => {
 468             intrinsic_args!(fx, args => (vtable); intrinsic);
 469             let vtable = vtable.load_scalar(fx);
 470
 471             let align = crate::vtable::min_align_of_obj(fx, vtable);
 472             ret.write_cvalue(fx, CValue::by_val(align, usize_layout));
 473         }
 474
 475         sym::exact_div => {
 476             intrinsic_args!(fx, args => (x, y); intrinsic);
 477
 478             // FIXME trap on inexact
 479             let res = crate::num::codegen_int_binop(fx, BinOp::Div, x, y);
 480             ret.write_cvalue(fx, res);
 481         }
 482         sym::saturating_add | sym::saturating_sub => {
 483             intrinsic_args!(fx, args => (lhs, rhs); intrinsic);
 484
 485             assert_eq!(lhs.layout().ty, rhs.layout().ty);
 486             let bin_op = match intrinsic {
 487                 sym::saturating_add => BinOp::Add,
 488                 sym::saturating_sub => BinOp::Sub,
 489                 _ => unreachable!(),
 490             };
 491
 492             let res = crate::num::codegen_saturating_int_binop(fx, bin_op, lhs, rhs);
 493             ret.write_cvalue(fx, res);
 494         }
 495         sym::rotate_left => {
 496             intrinsic_args!(fx, args => (x, y); intrinsic);
 497             let y = y.load_scalar(fx);
 498
 499             let layout = x.layout();
 500             let x = x.load_scalar(fx);
 501             let res = fx.bcx.ins().rotl(x, y);
 502             ret.write_cvalue(fx, CValue::by_val(res, layout));
 503         }
 504         sym::rotate_right => {
 505             intrinsic_args!(fx, args => (x, y); intrinsic);
 506             let y = y.load_scalar(fx);
 507
 508             let layout = x.layout();
 509             let x = x.load_scalar(fx);
 510             let res = fx.bcx.ins().rotr(x, y);
 511             ret.write_cvalue(fx, CValue::by_val(res, layout));
 512         }
 513
 514         // The only difference between offset and arith_offset is regarding UB. Because Cranelift
 515         // doesn't have UB both are codegen'ed the same way
 516         sym::arith_offset => {
 517             intrinsic_args!(fx, args => (base, offset); intrinsic);
 518             let offset = offset.load_scalar(fx);
 519
 520             let pointee_ty = base.layout().ty.builtin_deref(true).unwrap().ty;
 521             let pointee_size = fx.layout_of(pointee_ty).size.bytes();
 522             let ptr_diff = if pointee_size != 1 {
 523                 fx.bcx.ins().imul_imm(offset, pointee_size as i64)
 524             } else {
 525                 offset
 526             };
 527             let base_val = base.load_scalar(fx);
 528             let res = fx.bcx.ins().iadd(base_val, ptr_diff);
 529             ret.write_cvalue(fx, CValue::by_val(res, base.layout()));
 530         }
 531
 532         sym::ptr_mask => {
 533             intrinsic_args!(fx, args => (ptr, mask); intrinsic);
 534             let ptr = ptr.load_scalar(fx);
 535             let mask = mask.load_scalar(fx);
 536             fx.bcx.ins().band(ptr, mask);
 537         }
 538
 539         sym::write_bytes | sym::volatile_set_memory => {
 540             intrinsic_args!(fx, args => (dst, val, count); intrinsic);
 541             let val = val.load_scalar(fx);
 542             let count = count.load_scalar(fx);
 543
 544             let pointee_ty = dst.layout().ty.builtin_deref(true).unwrap().ty;
 545             let pointee_size = fx.layout_of(pointee_ty).size.bytes();
 546             let count = if pointee_size != 1 {
 547                 fx.bcx.ins().imul_imm(count, pointee_size as i64)
 548             } else {
 549                 count
 550             };
 551             let dst_ptr = dst.load_scalar(fx);
 552             // FIXME make the memset actually volatile when switching to emit_small_memset
 553             // FIXME use emit_small_memset
 554             fx.bcx.call_memset(fx.target_config, dst_ptr, val, count);
 555         }
 556         sym::ctlz | sym::ctlz_nonzero => {
 557             intrinsic_args!(fx, args => (arg); intrinsic);
 558             let val = arg.load_scalar(fx);
 559
 560             // FIXME trap on `ctlz_nonzero` with zero arg.
 561             let res = fx.bcx.ins().clz(val);
 562             let res = CValue::by_val(res, arg.layout());
 563             ret.write_cvalue(fx, res);
 564         }
 565         sym::cttz | sym::cttz_nonzero => {
 566             intrinsic_args!(fx, args => (arg); intrinsic);
 567             let val = arg.load_scalar(fx);
 568
 569             // FIXME trap on `cttz_nonzero` with zero arg.
 570             let res = fx.bcx.ins().ctz(val);
 571             let res = CValue::by_val(res, arg.layout());
 572             ret.write_cvalue(fx, res);
 573         }
 574         sym::ctpop => {
 575             intrinsic_args!(fx, args => (arg); intrinsic);
 576             let val = arg.load_scalar(fx);
 577
 578             let res = fx.bcx.ins().popcnt(val);
 579             let res = CValue::by_val(res, arg.layout());
 580             ret.write_cvalue(fx, res);
 581         }
 582         sym::bitreverse => {
 583             intrinsic_args!(fx, args => (arg); intrinsic);
 584             let val = arg.load_scalar(fx);
 585
 586             let res = fx.bcx.ins().bitrev(val);
 587             let res = CValue::by_val(res, arg.layout());
 588             ret.write_cvalue(fx, res);
 589         }
 590         sym::bswap => {
 591             intrinsic_args!(fx, args => (arg); intrinsic);
 592             let val = arg.load_scalar(fx);
 593
 594             let res = if fx.bcx.func.dfg.value_type(val) == types::I8 {
 595                 val
 596             } else {
 597                 fx.bcx.ins().bswap(val)
 598             };
 599             let res = CValue::by_val(res, arg.layout());
 600             ret.write_cvalue(fx, res);
 601         }
 602         sym::assert_inhabited | sym::assert_zero_valid | sym::assert_mem_uninitialized_valid => {
 603             intrinsic_args!(fx, args => (); intrinsic);
 604
 605             let ty = generic_args.type_at(0);
 606
 607             let requirement = ValidityRequirement::from_intrinsic(intrinsic);
 608
 609             if let Some(requirement) = requirement {
 610                 let do_panic = !fx
 611                     .tcx
 612                     .check_validity_requirement((requirement, fx.param_env().and(ty)))
 613                     .expect("expect to have layout during codegen");
 614
 615                 if do_panic {
 616                     let layout = fx.layout_of(ty);
 617                     let msg_str = with_no_visible_paths!({
 618                         with_no_trimmed_paths!({
 619                             if layout.abi.is_uninhabited() {
 620                                 // Use this error even for the other intrinsics as it is more precise.
 621                                 format!("attempted to instantiate uninhabited type `{}`", ty)
 622                             } else if intrinsic == sym::assert_zero_valid {
 623                                 format!(
 624                                     "attempted to zero-initialize type `{}`, which is invalid",
 625                                     ty
 626                                 )
 627                             } else {
 628                                 format!(
 629                                     "attempted to leave type `{}` uninitialized, which is invalid",
 630                                     ty
 631                                 )
 632                             }
 633                         })
 634                     });
 635                     crate::base::codegen_panic_nounwind(fx, &msg_str, source_info);
 636                     return;
 637                 }
 638             }
 639         }
 640
 641         sym::volatile_load | sym::unaligned_volatile_load => {
 642             intrinsic_args!(fx, args => (ptr); intrinsic);
 643
 644             // Cranelift treats loads as volatile by default
 645             // FIXME correctly handle unaligned_volatile_load
 646             let inner_layout = fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty);
 647             let val = CValue::by_ref(Pointer::new(ptr.load_scalar(fx)), inner_layout);
 648             ret.write_cvalue(fx, val);
 649         }
 650         sym::volatile_store | sym::unaligned_volatile_store | sym::nontemporal_store => {
 651             intrinsic_args!(fx, args => (ptr, val); intrinsic);
 652             let ptr = ptr.load_scalar(fx);
 653
 654             // Cranelift treats stores as volatile by default
 655             // FIXME correctly handle unaligned_volatile_store
 656             // FIXME actually do nontemporal stores if requested
 657             let dest = CPlace::for_ptr(Pointer::new(ptr), val.layout());
 658             dest.write_cvalue(fx, val);
 659         }
 660
 661         sym::pref_align_of
 662         | sym::needs_drop
 663         | sym::type_id
 664         | sym::type_name
 665         | sym::variant_count => {
 666             intrinsic_args!(fx, args => (); intrinsic);
 667
 668             let const_val =
 669                 fx.tcx.const_eval_instance(ParamEnv::reveal_all(), instance, None).unwrap();
 670             let val = crate::constant::codegen_const_value(fx, const_val, ret.layout().ty);
 671             ret.write_cvalue(fx, val);
 672         }
 673
 674         sym::ptr_offset_from | sym::ptr_offset_from_unsigned => {
 675             intrinsic_args!(fx, args => (ptr, base); intrinsic);
 676             let ptr = ptr.load_scalar(fx);
 677             let base = base.load_scalar(fx);
 678             let ty = generic_args.type_at(0);
 679
 680             let pointee_size: u64 = fx.layout_of(ty).size.bytes();
 681             let diff_bytes = fx.bcx.ins().isub(ptr, base);
 682             // FIXME this can be an exact division.
 683             let val = if intrinsic == sym::ptr_offset_from_unsigned {
 684                 let usize_layout = fx.layout_of(fx.tcx.types.usize);
 685                 // Because diff_bytes ULE isize::MAX, this would be fine as signed,
 686                 // but unsigned is slightly easier to codegen, so might as well.
 687                 CValue::by_val(fx.bcx.ins().udiv_imm(diff_bytes, pointee_size as i64), usize_layout)
 688             } else {
 689                 let isize_layout = fx.layout_of(fx.tcx.types.isize);
 690                 CValue::by_val(fx.bcx.ins().sdiv_imm(diff_bytes, pointee_size as i64), isize_layout)
 691             };
 692             ret.write_cvalue(fx, val);
 693         }
 694
 695         sym::ptr_guaranteed_cmp => {
 696             intrinsic_args!(fx, args => (a, b); intrinsic);
 697
 698             let val = crate::num::codegen_ptr_binop(fx, BinOp::Eq, a, b).load_scalar(fx);
 699             ret.write_cvalue(fx, CValue::by_val(val, fx.layout_of(fx.tcx.types.u8)));
 700         }
 701
 702         sym::caller_location => {
 703             intrinsic_args!(fx, args => (); intrinsic);
 704
 705             let caller_location = fx.get_caller_location(source_info);
 706             ret.write_cvalue(fx, caller_location);
 707         }
 708
 709         _ if intrinsic.as_str().starts_with("atomic_fence") => {
 710             intrinsic_args!(fx, args => (); intrinsic);
 711
 712             fx.bcx.ins().fence();
 713         }
 714         _ if intrinsic.as_str().starts_with("atomic_singlethreadfence") => {
 715             intrinsic_args!(fx, args => (); intrinsic);
 716
 717             // FIXME use a compiler fence once Cranelift supports it
 718             fx.bcx.ins().fence();
 719         }
 720         _ if intrinsic.as_str().starts_with("atomic_load") => {
 721             intrinsic_args!(fx, args => (ptr); intrinsic);
 722             let ptr = ptr.load_scalar(fx);
 723
 724             let ty = generic_args.type_at(0);
 725             match ty.kind() {
 726                 ty::Uint(UintTy::U128) | ty::Int(IntTy::I128) => {
 727                     // FIXME implement 128bit atomics
 728                     if fx.tcx.is_compiler_builtins(LOCAL_CRATE) {
 729                         // special case for compiler-builtins to avoid having to patch it
 730                         crate::trap::trap_unimplemented(fx, "128bit atomics not yet supported");
 731                         return;
 732                     } else {
 733                         fx.tcx
 734                             .sess
 735                             .span_fatal(source_info.span, "128bit atomics not yet supported");
 736                     }
 737                 }
 738                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
 739                 _ => {
 740                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, ty);
 741                     return;
 742                 }
 743             }
 744             let clif_ty = fx.clif_type(ty).unwrap();
 745
 746             let val = fx.bcx.ins().atomic_load(clif_ty, MemFlags::trusted(), ptr);
 747
 748             let val = CValue::by_val(val, fx.layout_of(ty));
 749             ret.write_cvalue(fx, val);
 750         }
 751         _ if intrinsic.as_str().starts_with("atomic_store") => {
 752             intrinsic_args!(fx, args => (ptr, val); intrinsic);
 753             let ptr = ptr.load_scalar(fx);
 754
 755             let ty = generic_args.type_at(0);
 756             match ty.kind() {
 757                 ty::Uint(UintTy::U128) | ty::Int(IntTy::I128) => {
 758                     // FIXME implement 128bit atomics
 759                     if fx.tcx.is_compiler_builtins(LOCAL_CRATE) {
 760                         // special case for compiler-builtins to avoid having to patch it
 761                         crate::trap::trap_unimplemented(fx, "128bit atomics not yet supported");
 762                         return;
 763                     } else {
 764                         fx.tcx
 765                             .sess
 766                             .span_fatal(source_info.span, "128bit atomics not yet supported");
 767                     }
 768                 }
 769                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
 770                 _ => {
 771                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, ty);
 772                     return;
 773                 }
 774             }
 775
 776             let val = val.load_scalar(fx);
 777
 778             fx.bcx.ins().atomic_store(MemFlags::trusted(), val, ptr);
 779         }
 780         _ if intrinsic.as_str().starts_with("atomic_xchg") => {
 781             intrinsic_args!(fx, args => (ptr, new); intrinsic);
 782             let ptr = ptr.load_scalar(fx);
 783
 784             let layout = new.layout();
 785             match layout.ty.kind() {
 786                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
 787                 _ => {
 788                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
 789                     return;
 790                 }
 791             }
 792             let ty = fx.clif_type(layout.ty).unwrap();
 793
 794             let new = new.load_scalar(fx);
 795
 796             let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Xchg, ptr, new);
 797
 798             let old = CValue::by_val(old, layout);
 799             ret.write_cvalue(fx, old);
 800         }
 801         _ if intrinsic.as_str().starts_with("atomic_cxchg") => {
 802             // both atomic_cxchg_* and atomic_cxchgweak_*
 803             intrinsic_args!(fx, args => (ptr, test_old, new); intrinsic);
 804             let ptr = ptr.load_scalar(fx);
 805
 806             let layout = new.layout();
 807             match layout.ty.kind() {
 808                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
 809                 _ => {
 810                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
 811                     return;
 812                 }
 813             }
 814
 815             let test_old = test_old.load_scalar(fx);
 816             let new = new.load_scalar(fx);
 817
 818             let old = fx.bcx.ins().atomic_cas(MemFlags::trusted(), ptr, test_old, new);
 819             let is_eq = fx.bcx.ins().icmp(IntCC::Equal, old, test_old);
 820
 821             let ret_val = CValue::by_val_pair(old, is_eq, ret.layout());
 822             ret.write_cvalue(fx, ret_val)
 823         }
 824
 825         _ if intrinsic.as_str().starts_with("atomic_xadd") => {
 826             intrinsic_args!(fx, args => (ptr, amount); intrinsic);
 827             let ptr = ptr.load_scalar(fx);
 828
 829             let layout = amount.layout();
 830             match layout.ty.kind() {
 831                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
 832                 _ => {
 833                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
 834                     return;
 835                 }
 836             }
 837             let ty = fx.clif_type(layout.ty).unwrap();
 838
 839             let amount = amount.load_scalar(fx);
 840
 841             let old =
 842                 fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Add, ptr, amount);
 843
 844             let old = CValue::by_val(old, layout);
 845             ret.write_cvalue(fx, old);
 846         }
 847         _ if intrinsic.as_str().starts_with("atomic_xsub") => {
 848             intrinsic_args!(fx, args => (ptr, amount); intrinsic);
 849             let ptr = ptr.load_scalar(fx);
 850
 851             let layout = amount.layout();
 852             match layout.ty.kind() {
 853                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
 854                 _ => {
 855                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
 856                     return;
 857                 }
 858             }
 859             let ty = fx.clif_type(layout.ty).unwrap();
 860
 861             let amount = amount.load_scalar(fx);
 862
 863             let old =
 864                 fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Sub, ptr, amount);
 865
 866             let old = CValue::by_val(old, layout);
 867             ret.write_cvalue(fx, old);
 868         }
 869         _ if intrinsic.as_str().starts_with("atomic_and") => {
 870             intrinsic_args!(fx, args => (ptr, src); intrinsic);
 871             let ptr = ptr.load_scalar(fx);
 872
 873             let layout = src.layout();
 874             match layout.ty.kind() {
 875                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
 876                 _ => {
 877                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
 878                     return;
 879                 }
 880             }
 881             let ty = fx.clif_type(layout.ty).unwrap();
 882
 883             let src = src.load_scalar(fx);
 884
 885             let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::And, ptr, src);
 886
 887             let old = CValue::by_val(old, layout);
 888             ret.write_cvalue(fx, old);
 889         }
 890         _ if intrinsic.as_str().starts_with("atomic_or") => {
 891             intrinsic_args!(fx, args => (ptr, src); intrinsic);
 892             let ptr = ptr.load_scalar(fx);
 893
 894             let layout = src.layout();
 895             match layout.ty.kind() {
 896                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
 897                 _ => {
 898                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
 899                     return;
 900                 }
 901             }
 902             let ty = fx.clif_type(layout.ty).unwrap();
 903
 904             let src = src.load_scalar(fx);
 905
 906             let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Or, ptr, src);
 907
 908             let old = CValue::by_val(old, layout);
 909             ret.write_cvalue(fx, old);
 910         }
 911         _ if intrinsic.as_str().starts_with("atomic_xor") => {
 912             intrinsic_args!(fx, args => (ptr, src); intrinsic);
 913             let ptr = ptr.load_scalar(fx);
 914
 915             let layout = src.layout();
 916             match layout.ty.kind() {
 917                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
 918                 _ => {
 919                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
 920                     return;
 921                 }
 922             }
 923             let ty = fx.clif_type(layout.ty).unwrap();
 924
 925             let src = src.load_scalar(fx);
 926
 927             let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Xor, ptr, src);
 928
 929             let old = CValue::by_val(old, layout);
 930             ret.write_cvalue(fx, old);
 931         }
 932         _ if intrinsic.as_str().starts_with("atomic_nand") => {
 933             intrinsic_args!(fx, args => (ptr, src); intrinsic);
 934             let ptr = ptr.load_scalar(fx);
 935
 936             let layout = src.layout();
 937             match layout.ty.kind() {
 938                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
 939                 _ => {
 940                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
 941                     return;
 942                 }
 943             }
 944             let ty = fx.clif_type(layout.ty).unwrap();
 945
 946             let src = src.load_scalar(fx);
 947
 948             let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Nand, ptr, src);
 949
 950             let old = CValue::by_val(old, layout);
 951             ret.write_cvalue(fx, old);
 952         }
 953         _ if intrinsic.as_str().starts_with("atomic_max") => {
 954             intrinsic_args!(fx, args => (ptr, src); intrinsic);
 955             let ptr = ptr.load_scalar(fx);
 956
 957             let layout = src.layout();
 958             match layout.ty.kind() {
 959                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
 960                 _ => {
 961                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
 962                     return;
 963                 }
 964             }
 965             let ty = fx.clif_type(layout.ty).unwrap();
 966
 967             let src = src.load_scalar(fx);
 968
 969             let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Smax, ptr, src);
 970
 971             let old = CValue::by_val(old, layout);
 972             ret.write_cvalue(fx, old);
 973         }
 974         _ if intrinsic.as_str().starts_with("atomic_umax") => {
 975             intrinsic_args!(fx, args => (ptr, src); intrinsic);
 976             let ptr = ptr.load_scalar(fx);
 977
 978             let layout = src.layout();
 979             match layout.ty.kind() {
 980                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
 981                 _ => {
 982                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
 983                     return;
 984                 }
 985             }
 986             let ty = fx.clif_type(layout.ty).unwrap();
 987
 988             let src = src.load_scalar(fx);
 989
 990             let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Umax, ptr, src);
 991
 992             let old = CValue::by_val(old, layout);
 993             ret.write_cvalue(fx, old);
 994         }
 995         _ if intrinsic.as_str().starts_with("atomic_min") => {
 996             intrinsic_args!(fx, args => (ptr, src); intrinsic);
 997             let ptr = ptr.load_scalar(fx);
 998
 999             let layout = src.layout();
1000             match layout.ty.kind() {
1001                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
1002                 _ => {
1003                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
1004                     return;
1005                 }
1006             }
1007             let ty = fx.clif_type(layout.ty).unwrap();
1008
1009             let src = src.load_scalar(fx);
1010
1011             let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Smin, ptr, src);
1012
1013             let old = CValue::by_val(old, layout);
1014             ret.write_cvalue(fx, old);
1015         }
1016         _ if intrinsic.as_str().starts_with("atomic_umin") => {
1017             intrinsic_args!(fx, args => (ptr, src); intrinsic);
1018             let ptr = ptr.load_scalar(fx);
1019
1020             let layout = src.layout();
1021             match layout.ty.kind() {
1022                 ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
1023                 _ => {
1024                     report_atomic_type_validation_error(fx, intrinsic, source_info.span, layout.ty);
1025                     return;
1026                 }
1027             }
1028             let ty = fx.clif_type(layout.ty).unwrap();
1029
1030             let src = src.load_scalar(fx);
1031
1032             let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Umin, ptr, src);
1033
1034             let old = CValue::by_val(old, layout);
1035             ret.write_cvalue(fx, old);
1036         }
1037
1038         sym::minnumf32 => {
1039             intrinsic_args!(fx, args => (a, b); intrinsic);
1040             let a = a.load_scalar(fx);
1041             let b = b.load_scalar(fx);
1042
1043             let val = crate::num::codegen_float_min(fx, a, b);
1044             let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f32));
1045             ret.write_cvalue(fx, val);
1046         }
1047         sym::minnumf64 => {
1048             intrinsic_args!(fx, args => (a, b); intrinsic);
1049             let a = a.load_scalar(fx);
1050             let b = b.load_scalar(fx);
1051
1052             let val = crate::num::codegen_float_min(fx, a, b);
1053             let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f64));
1054             ret.write_cvalue(fx, val);
1055         }
1056         sym::maxnumf32 => {
1057             intrinsic_args!(fx, args => (a, b); intrinsic);
1058             let a = a.load_scalar(fx);
1059             let b = b.load_scalar(fx);
1060
1061             let val = crate::num::codegen_float_max(fx, a, b);
1062             let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f32));
1063             ret.write_cvalue(fx, val);
1064         }
1065         sym::maxnumf64 => {
1066             intrinsic_args!(fx, args => (a, b); intrinsic);
1067             let a = a.load_scalar(fx);
1068             let b = b.load_scalar(fx);
1069
1070             let val = crate::num::codegen_float_max(fx, a, b);
1071             let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f64));
1072             ret.write_cvalue(fx, val);
1073         }
1074
1075         kw::Try => {
1076             intrinsic_args!(fx, args => (f, data, catch_fn); intrinsic);
1077             let f = f.load_scalar(fx);
1078             let data = data.load_scalar(fx);
1079             let _catch_fn = catch_fn.load_scalar(fx);
1080
1081             // FIXME once unwinding is supported, change this to actually catch panics
1082             let f_sig = fx.bcx.func.import_signature(Signature {
1083                 call_conv: fx.target_config.default_call_conv,
1084                 params: vec![AbiParam::new(pointer_ty(fx.tcx))],
1085                 returns: vec![],
1086             });
1087
1088             fx.bcx.ins().call_indirect(f_sig, f, &[data]);
1089
1090             let layout = fx.layout_of(fx.tcx.types.i32);
1091             let ret_val = CValue::by_val(fx.bcx.ins().iconst(types::I32, 0), layout);
1092             ret.write_cvalue(fx, ret_val);
1093         }
1094
1095         sym::fadd_fast | sym::fsub_fast | sym::fmul_fast | sym::fdiv_fast | sym::frem_fast => {
1096             intrinsic_args!(fx, args => (x, y); intrinsic);
1097
1098             let res = crate::num::codegen_float_binop(
1099                 fx,
1100                 match intrinsic {
1101                     sym::fadd_fast => BinOp::Add,
1102                     sym::fsub_fast => BinOp::Sub,
1103                     sym::fmul_fast => BinOp::Mul,
1104                     sym::fdiv_fast => BinOp::Div,
1105                     sym::frem_fast => BinOp::Rem,
1106                     _ => unreachable!(),
1107                 },
1108                 x,
1109                 y,
1110             );
1111             ret.write_cvalue(fx, res);
1112         }
1113         sym::float_to_int_unchecked => {
1114             intrinsic_args!(fx, args => (f); intrinsic);
1115             let f = f.load_scalar(fx);
1116
1117             let res = crate::cast::clif_int_or_float_cast(
1118                 fx,
1119                 f,
1120                 false,
1121                 fx.clif_type(ret.layout().ty).unwrap(),
1122                 type_sign(ret.layout().ty),
1123             );
1124             ret.write_cvalue(fx, CValue::by_val(res, ret.layout()));
1125         }
1126
1127         sym::raw_eq => {
1128             intrinsic_args!(fx, args => (lhs_ref, rhs_ref); intrinsic);
1129             let lhs_ref = lhs_ref.load_scalar(fx);
1130             let rhs_ref = rhs_ref.load_scalar(fx);
1131
1132             let size = fx.layout_of(generic_args.type_at(0)).layout.size();
1133             // FIXME add and use emit_small_memcmp
1134             let is_eq_value = if size == Size::ZERO {
1135                 // No bytes means they're trivially equal
1136                 fx.bcx.ins().iconst(types::I8, 1)
1137             } else if let Some(clty) = size.bits().try_into().ok().and_then(Type::int) {
1138                 // Can't use `trusted` for these loads; they could be unaligned.
1139                 let mut flags = MemFlags::new();
1140                 flags.set_notrap();
1141                 let lhs_val = fx.bcx.ins().load(clty, flags, lhs_ref, 0);
1142                 let rhs_val = fx.bcx.ins().load(clty, flags, rhs_ref, 0);
1143                 fx.bcx.ins().icmp(IntCC::Equal, lhs_val, rhs_val)
1144             } else {
1145                 // Just call `memcmp` (like slices do in core) when the
1146                 // size is too large or it's not a power-of-two.
1147                 let signed_bytes = i64::try_from(size.bytes()).unwrap();
1148                 let bytes_val = fx.bcx.ins().iconst(fx.pointer_type, signed_bytes);
1149                 let params = vec![AbiParam::new(fx.pointer_type); 3];
1150                 let returns = vec![AbiParam::new(types::I32)];
1151                 let args = &[lhs_ref, rhs_ref, bytes_val];
1152                 let cmp = fx.lib_call("memcmp", params, returns, args)[0];
1153                 fx.bcx.ins().icmp_imm(IntCC::Equal, cmp, 0)
1154             };
1155             ret.write_cvalue(fx, CValue::by_val(is_eq_value, ret.layout()));
1156         }
1157
1158         sym::compare_bytes => {
1159             intrinsic_args!(fx, args => (lhs_ptr, rhs_ptr, bytes_val); intrinsic);
1160             let lhs_ptr = lhs_ptr.load_scalar(fx);
1161             let rhs_ptr = rhs_ptr.load_scalar(fx);
1162             let bytes_val = bytes_val.load_scalar(fx);
1163
1164             let params = vec![AbiParam::new(fx.pointer_type); 3];
1165             let returns = vec![AbiParam::new(types::I32)];
1166             let args = &[lhs_ptr, rhs_ptr, bytes_val];
1167             // Here we assume that the `memcmp` provided by the target is a NOP for size 0.
1168             let cmp = fx.lib_call("memcmp", params, returns, args)[0];
1169             ret.write_cvalue(fx, CValue::by_val(cmp, ret.layout()));
1170         }
1171
1172         sym::const_allocate => {
1173             intrinsic_args!(fx, args => (_size, _align); intrinsic);
1174
1175             // returns a null pointer at runtime.
1176             let null = fx.bcx.ins().iconst(fx.pointer_type, 0);
1177             ret.write_cvalue(fx, CValue::by_val(null, ret.layout()));
1178         }
1179
1180         sym::const_deallocate => {
1181             intrinsic_args!(fx, args => (_ptr, _size, _align); intrinsic);
1182             // nop at runtime.
1183         }
1184
1185         sym::black_box => {
1186             intrinsic_args!(fx, args => (a); intrinsic);
1187
1188             // FIXME implement black_box semantics
1189             ret.write_cvalue(fx, a);
1190         }
1191
1192         // FIXME implement variadics in cranelift
1193         sym::va_copy | sym::va_arg | sym::va_end => {
1194             fx.tcx.sess.span_fatal(
1195                 source_info.span,
1196                 "Defining variadic functions is not yet supported by Cranelift",
1197             );
1198         }
1199
1200         _ => {
1201             fx.tcx
1202                 .sess
1203                 .span_fatal(source_info.span, format!("unsupported intrinsic {}", intrinsic));
1204         }
1205     }
1206
1207     let ret_block = fx.get_block(destination.unwrap());
1208     fx.bcx.ins().jump(ret_block, &[]);
1209 }