src/librustc_trans/trans/intrinsic.rs

   1 // Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
   2 // file at the top-level directory of this distribution and at
   3 // http://rust-lang.org/COPYRIGHT.
   4 //
   5 // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
   6 // http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
   7 // <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
   8 // option. This file may not be copied, modified, or distributed
   9 // except according to those terms.
  10
  11 #![allow(non_upper_case_globals)]
  12
  13 use arena::TypedArena;
  14 use intrinsics::{self, Intrinsic};
  15 use libc;
  16 use llvm;
  17 use llvm::{SequentiallyConsistent, Acquire, Release, AtomicXchg, ValueRef, TypeKind};
  18 use middle::subst;
  19 use middle::subst::FnSpace;
  20 use trans::adt;
  21 use trans::attributes;
  22 use trans::base::*;
  23 use trans::build::*;
  24 use trans::callee;
  25 use trans::cleanup;
  26 use trans::cleanup::CleanupMethods;
  27 use trans::common::*;
  28 use trans::consts;
  29 use trans::datum::*;
  30 use trans::debuginfo::DebugLoc;
  31 use trans::declare;
  32 use trans::expr;
  33 use trans::glue;
  34 use trans::type_of::*;
  35 use trans::type_of;
  36 use trans::machine;
  37 use trans::machine::llsize_of;
  38 use trans::type_::Type;
  39 use middle::ty::{self, Ty, HasTypeFlags};
  40 use middle::subst::Substs;
  41 use rustc_front::hir;
  42 use syntax::abi::{self, RustIntrinsic};
  43 use syntax::ast;
  44 use syntax::ptr::P;
  45 use syntax::parse::token;
  46
  47 use std::cmp::Ordering;
  48
  49 pub fn get_simple_intrinsic(ccx: &CrateContext, item: &hir::ForeignItem) -> Option<ValueRef> {
  50     let name = match &*item.ident.name.as_str() {
  51         "sqrtf32" => "llvm.sqrt.f32",
  52         "sqrtf64" => "llvm.sqrt.f64",
  53         "powif32" => "llvm.powi.f32",
  54         "powif64" => "llvm.powi.f64",
  55         "sinf32" => "llvm.sin.f32",
  56         "sinf64" => "llvm.sin.f64",
  57         "cosf32" => "llvm.cos.f32",
  58         "cosf64" => "llvm.cos.f64",
  59         "powf32" => "llvm.pow.f32",
  60         "powf64" => "llvm.pow.f64",
  61         "expf32" => "llvm.exp.f32",
  62         "expf64" => "llvm.exp.f64",
  63         "exp2f32" => "llvm.exp2.f32",
  64         "exp2f64" => "llvm.exp2.f64",
  65         "logf32" => "llvm.log.f32",
  66         "logf64" => "llvm.log.f64",
  67         "log10f32" => "llvm.log10.f32",
  68         "log10f64" => "llvm.log10.f64",
  69         "log2f32" => "llvm.log2.f32",
  70         "log2f64" => "llvm.log2.f64",
  71         "fmaf32" => "llvm.fma.f32",
  72         "fmaf64" => "llvm.fma.f64",
  73         "fabsf32" => "llvm.fabs.f32",
  74         "fabsf64" => "llvm.fabs.f64",
  75         "copysignf32" => "llvm.copysign.f32",
  76         "copysignf64" => "llvm.copysign.f64",
  77         "floorf32" => "llvm.floor.f32",
  78         "floorf64" => "llvm.floor.f64",
  79         "ceilf32" => "llvm.ceil.f32",
  80         "ceilf64" => "llvm.ceil.f64",
  81         "truncf32" => "llvm.trunc.f32",
  82         "truncf64" => "llvm.trunc.f64",
  83         "rintf32" => "llvm.rint.f32",
  84         "rintf64" => "llvm.rint.f64",
  85         "nearbyintf32" => "llvm.nearbyint.f32",
  86         "nearbyintf64" => "llvm.nearbyint.f64",
  87         "roundf32" => "llvm.round.f32",
  88         "roundf64" => "llvm.round.f64",
  89         "ctpop8" => "llvm.ctpop.i8",
  90         "ctpop16" => "llvm.ctpop.i16",
  91         "ctpop32" => "llvm.ctpop.i32",
  92         "ctpop64" => "llvm.ctpop.i64",
  93         "bswap16" => "llvm.bswap.i16",
  94         "bswap32" => "llvm.bswap.i32",
  95         "bswap64" => "llvm.bswap.i64",
  96         "assume" => "llvm.assume",
  97         _ => return None
  98     };
  99     Some(ccx.get_intrinsic(&name))
 100 }
 101
 102 /// Performs late verification that intrinsics are used correctly. At present,
 103 /// the only intrinsic that needs such verification is `transmute`.
 104 pub fn check_intrinsics(ccx: &CrateContext) {
 105     let mut last_failing_id = None;
 106     for transmute_restriction in ccx.tcx().transmute_restrictions.borrow().iter() {
 107         // Sometimes, a single call to transmute will push multiple
 108         // type pairs to test in order to exhaustively test the
 109         // possibility around a type parameter. If one of those fails,
 110         // there is no sense reporting errors on the others.
 111         if last_failing_id == Some(transmute_restriction.id) {
 112             continue;
 113         }
 114
 115         debug!("transmute_restriction: {:?}", transmute_restriction);
 116
 117         assert!(!transmute_restriction.substituted_from.has_param_types());
 118         assert!(!transmute_restriction.substituted_to.has_param_types());
 119
 120         let llfromtype = type_of::sizing_type_of(ccx,
 121                                                  transmute_restriction.substituted_from);
 122         let lltotype = type_of::sizing_type_of(ccx,
 123                                                transmute_restriction.substituted_to);
 124         let from_type_size = machine::llbitsize_of_real(ccx, llfromtype);
 125         let to_type_size = machine::llbitsize_of_real(ccx, lltotype);
 126         if from_type_size != to_type_size {
 127             last_failing_id = Some(transmute_restriction.id);
 128
 129             if transmute_restriction.original_from != transmute_restriction.substituted_from {
 130                 ccx.sess().span_err(
 131                     transmute_restriction.span,
 132                     &format!("transmute called on types with potentially different sizes: \
 133                               {} (could be {} bit{}) to {} (could be {} bit{})",
 134                              transmute_restriction.original_from,
 135                              from_type_size as usize,
 136                              if from_type_size == 1 {""} else {"s"},
 137                              transmute_restriction.original_to,
 138                              to_type_size as usize,
 139                              if to_type_size == 1 {""} else {"s"}));
 140             } else {
 141                 ccx.sess().span_err(
 142                     transmute_restriction.span,
 143                     &format!("transmute called on types with different sizes: \
 144                               {} ({} bit{}) to {} ({} bit{})",
 145                              transmute_restriction.original_from,
 146                              from_type_size as usize,
 147                              if from_type_size == 1 {""} else {"s"},
 148                              transmute_restriction.original_to,
 149                              to_type_size as usize,
 150                              if to_type_size == 1 {""} else {"s"}));
 151             }
 152         }
 153     }
 154     ccx.sess().abort_if_errors();
 155 }
 156
 157 /// Remember to add all intrinsics here, in librustc_typeck/check/mod.rs,
 158 /// and in libcore/intrinsics.rs; if you need access to any llvm intrinsics,
 159 /// add them to librustc_trans/trans/context.rs
 160 pub fn trans_intrinsic_call<'a, 'blk, 'tcx>(mut bcx: Block<'blk, 'tcx>,
 161                                             node: ast::NodeId,
 162                                             callee_ty: Ty<'tcx>,
 163                                             cleanup_scope: cleanup::CustomScopeIndex,
 164                                             args: callee::CallArgs<'a, 'tcx>,
 165                                             dest: expr::Dest,
 166                                             substs: subst::Substs<'tcx>,
 167                                             call_info: NodeIdAndSpan)
 168                                             -> Result<'blk, 'tcx> {
 169     let fcx = bcx.fcx;
 170     let ccx = fcx.ccx;
 171     let tcx = bcx.tcx();
 172
 173     let _icx = push_ctxt("trans_intrinsic_call");
 174
 175     let (arg_tys, ret_ty) = match callee_ty.sty {
 176         ty::TyBareFn(_, ref f) => {
 177             (bcx.tcx().erase_late_bound_regions(&f.sig.inputs()),
 178              bcx.tcx().erase_late_bound_regions(&f.sig.output()))
 179         }
 180         _ => panic!("expected bare_fn in trans_intrinsic_call")
 181     };
 182     let foreign_item = tcx.map.expect_foreign_item(node);
 183     let name = foreign_item.ident.name.as_str();
 184
 185     // For `transmute` we can just trans the input expr directly into dest
 186     if name == "transmute" {
 187         let llret_ty = type_of::type_of(ccx, ret_ty.unwrap());
 188         match args {
 189             callee::ArgExprs(arg_exprs) => {
 190                 assert_eq!(arg_exprs.len(), 1);
 191
 192                 let (in_type, out_type) = (*substs.types.get(FnSpace, 0),
 193                                            *substs.types.get(FnSpace, 1));
 194                 let llintype = type_of::type_of(ccx, in_type);
 195                 let llouttype = type_of::type_of(ccx, out_type);
 196
 197                 let in_type_size = machine::llbitsize_of_real(ccx, llintype);
 198                 let out_type_size = machine::llbitsize_of_real(ccx, llouttype);
 199
 200                 // This should be caught by the intrinsicck pass
 201                 assert_eq!(in_type_size, out_type_size);
 202
 203                 let nonpointer_nonaggregate = |llkind: TypeKind| -> bool {
 204                     use llvm::TypeKind::*;
 205                     match llkind {
 206                         Half | Float | Double | X86_FP80 | FP128 |
 207                             PPC_FP128 | Integer | Vector | X86_MMX => true,
 208                         _ => false
 209                     }
 210                 };
 211
 212                 // An approximation to which types can be directly cast via
 213                 // LLVM's bitcast.  This doesn't cover pointer -> pointer casts,
 214                 // but does, importantly, cover SIMD types.
 215                 let in_kind = llintype.kind();
 216                 let ret_kind = llret_ty.kind();
 217                 let bitcast_compatible =
 218                     (nonpointer_nonaggregate(in_kind) && nonpointer_nonaggregate(ret_kind)) || {
 219                         in_kind == TypeKind::Pointer && ret_kind == TypeKind::Pointer
 220                     };
 221
 222                 let dest = if bitcast_compatible {
 223                     // if we're here, the type is scalar-like (a primitive, a
 224                     // SIMD type or a pointer), and so can be handled as a
 225                     // by-value ValueRef and can also be directly bitcast to the
 226                     // target type.  Doing this special case makes conversions
 227                     // like `u32x4` -> `u64x2` much nicer for LLVM and so more
 228                     // efficient (these are done efficiently implicitly in C
 229                     // with the `__m128i` type and so this means Rust doesn't
 230                     // lose out there).
 231                     let expr = &*arg_exprs[0];
 232                     let datum = unpack_datum!(bcx, expr::trans(bcx, expr));
 233                     let datum = unpack_datum!(bcx, datum.to_rvalue_datum(bcx, "transmute_temp"));
 234                     let val = if datum.kind.is_by_ref() {
 235                         load_ty(bcx, datum.val, datum.ty)
 236                     } else {
 237                         from_arg_ty(bcx, datum.val, datum.ty)
 238                     };
 239
 240                     let cast_val = BitCast(bcx, val, llret_ty);
 241
 242                     match dest {
 243                         expr::SaveIn(d) => {
 244                             // this often occurs in a sequence like `Store(val,
 245                             // d); val2 = Load(d)`, so disappears easily.
 246                             Store(bcx, cast_val, d);
 247                         }
 248                         expr::Ignore => {}
 249                     }
 250                     dest
 251                 } else {
 252                     // The types are too complicated to do with a by-value
 253                     // bitcast, so pointer cast instead. We need to cast the
 254                     // dest so the types work out.
 255                     let dest = match dest {
 256                         expr::SaveIn(d) => expr::SaveIn(PointerCast(bcx, d, llintype.ptr_to())),
 257                         expr::Ignore => expr::Ignore
 258                     };
 259                     bcx = expr::trans_into(bcx, &*arg_exprs[0], dest);
 260                     dest
 261                 };
 262
 263                 fcx.scopes.borrow_mut().last_mut().unwrap().drop_non_lifetime_clean();
 264                 fcx.pop_and_trans_custom_cleanup_scope(bcx, cleanup_scope);
 265
 266                 return match dest {
 267                     expr::SaveIn(d) => Result::new(bcx, d),
 268                     expr::Ignore => Result::new(bcx, C_undef(llret_ty.ptr_to()))
 269                 };
 270
 271             }
 272
 273             _ => {
 274                 ccx.sess().bug("expected expr as argument for transmute");
 275             }
 276         }
 277     }
 278
 279     // For `move_val_init` we can evaluate the destination address
 280     // (the first argument) and then trans the source value (the
 281     // second argument) directly into the resulting destination
 282     // address.
 283     if name == "move_val_init" {
 284         if let callee::ArgExprs(ref exprs) = args {
 285             let (dest_expr, source_expr) = if exprs.len() != 2 {
 286                 ccx.sess().bug("expected two exprs as arguments for `move_val_init` intrinsic");
 287             } else {
 288                 (&exprs[0], &exprs[1])
 289             };
 290
 291             // evaluate destination address
 292             let dest_datum = unpack_datum!(bcx, expr::trans(bcx, dest_expr));
 293             let dest_datum = unpack_datum!(
 294                 bcx, dest_datum.to_rvalue_datum(bcx, "arg"));
 295             let dest_datum = unpack_datum!(
 296                 bcx, dest_datum.to_appropriate_datum(bcx));
 297
 298             // `expr::trans_into(bcx, expr, dest)` is equiv to
 299             //
 300             //    `trans(bcx, expr).store_to_dest(dest)`,
 301             //
 302             // which for `dest == expr::SaveIn(addr)`, is equivalent to:
 303             //
 304             //    `trans(bcx, expr).store_to(bcx, addr)`.
 305             let lldest = expr::Dest::SaveIn(dest_datum.val);
 306             bcx = expr::trans_into(bcx, source_expr, lldest);
 307
 308             let llresult = C_nil(ccx);
 309             fcx.pop_and_trans_custom_cleanup_scope(bcx, cleanup_scope);
 310
 311             return Result::new(bcx, llresult);
 312         } else {
 313             ccx.sess().bug("expected two exprs as arguments for `move_val_init` intrinsic");
 314         }
 315     }
 316
 317     let call_debug_location = DebugLoc::At(call_info.id, call_info.span);
 318
 319     // For `try` we need some custom control flow
 320     if &name[..] == "try" {
 321         if let callee::ArgExprs(ref exprs) = args {
 322             let (func, data) = if exprs.len() != 2 {
 323                 ccx.sess().bug("expected two exprs as arguments for \
 324                                 `try` intrinsic");
 325             } else {
 326                 (&exprs[0], &exprs[1])
 327             };
 328
 329             // translate arguments
 330             let func = unpack_datum!(bcx, expr::trans(bcx, func));
 331             let func = unpack_datum!(bcx, func.to_rvalue_datum(bcx, "func"));
 332             let data = unpack_datum!(bcx, expr::trans(bcx, data));
 333             let data = unpack_datum!(bcx, data.to_rvalue_datum(bcx, "data"));
 334
 335             let dest = match dest {
 336                 expr::SaveIn(d) => d,
 337                 expr::Ignore => alloc_ty(bcx, tcx.mk_mut_ptr(tcx.types.i8),
 338                                          "try_result"),
 339             };
 340
 341             // do the invoke
 342             bcx = try_intrinsic(bcx, func.val, data.val, dest,
 343                                 call_debug_location);
 344
 345             fcx.pop_and_trans_custom_cleanup_scope(bcx, cleanup_scope);
 346             return Result::new(bcx, dest);
 347         } else {
 348             ccx.sess().bug("expected two exprs as arguments for \
 349                             `try` intrinsic");
 350         }
 351     }
 352
 353     // save the actual AST arguments for later (some places need to do
 354     // const-evaluation on them)
 355     let expr_arguments = match args {
 356         callee::ArgExprs(args) => Some(args),
 357         _ => None,
 358     };
 359
 360     // Push the arguments.
 361     let mut llargs = Vec::new();
 362     bcx = callee::trans_args(bcx,
 363                              args,
 364                              callee_ty,
 365                              &mut llargs,
 366                              cleanup::CustomScope(cleanup_scope),
 367                              false,
 368                              RustIntrinsic);
 369
 370     fcx.scopes.borrow_mut().last_mut().unwrap().drop_non_lifetime_clean();
 371
 372     // These are the only intrinsic functions that diverge.
 373     if name == "abort" {
 374         let llfn = ccx.get_intrinsic(&("llvm.trap"));
 375         Call(bcx, llfn, &[], None, call_debug_location);
 376         fcx.pop_and_trans_custom_cleanup_scope(bcx, cleanup_scope);
 377         Unreachable(bcx);
 378         return Result::new(bcx, C_undef(Type::nil(ccx).ptr_to()));
 379     } else if &name[..] == "unreachable" {
 380         fcx.pop_and_trans_custom_cleanup_scope(bcx, cleanup_scope);
 381         Unreachable(bcx);
 382         return Result::new(bcx, C_nil(ccx));
 383     }
 384
 385     let ret_ty = match ret_ty {
 386         ty::FnConverging(ret_ty) => ret_ty,
 387         ty::FnDiverging => unreachable!()
 388     };
 389
 390     let llret_ty = type_of::type_of(ccx, ret_ty);
 391
 392     // Get location to store the result. If the user does
 393     // not care about the result, just make a stack slot
 394     let llresult = match dest {
 395         expr::SaveIn(d) => d,
 396         expr::Ignore => {
 397             if !type_is_zero_size(ccx, ret_ty) {
 398                 let llresult = alloc_ty(bcx, ret_ty, "intrinsic_result");
 399                 call_lifetime_start(bcx, llresult);
 400                 llresult
 401             } else {
 402                 C_undef(llret_ty.ptr_to())
 403             }
 404         }
 405     };
 406
 407     let simple = get_simple_intrinsic(ccx, &*foreign_item);
 408     let llval = match (simple, &*name) {
 409         (Some(llfn), _) => {
 410             Call(bcx, llfn, &llargs, None, call_debug_location)
 411         }
 412         (_, "breakpoint") => {
 413             let llfn = ccx.get_intrinsic(&("llvm.debugtrap"));
 414             Call(bcx, llfn, &[], None, call_debug_location)
 415         }
 416         (_, "size_of") => {
 417             let tp_ty = *substs.types.get(FnSpace, 0);
 418             let lltp_ty = type_of::type_of(ccx, tp_ty);
 419             C_uint(ccx, machine::llsize_of_alloc(ccx, lltp_ty))
 420         }
 421         (_, "size_of_val") => {
 422             let tp_ty = *substs.types.get(FnSpace, 0);
 423             if !type_is_sized(tcx, tp_ty) {
 424                 let (llsize, _) = glue::size_and_align_of_dst(bcx, tp_ty, llargs[1]);
 425                 llsize
 426             } else {
 427                 let lltp_ty = type_of::type_of(ccx, tp_ty);
 428                 C_uint(ccx, machine::llsize_of_alloc(ccx, lltp_ty))
 429             }
 430         }
 431         (_, "min_align_of") => {
 432             let tp_ty = *substs.types.get(FnSpace, 0);
 433             C_uint(ccx, type_of::align_of(ccx, tp_ty))
 434         }
 435         (_, "min_align_of_val") => {
 436             let tp_ty = *substs.types.get(FnSpace, 0);
 437             if !type_is_sized(tcx, tp_ty) {
 438                 let (_, llalign) = glue::size_and_align_of_dst(bcx, tp_ty, llargs[1]);
 439                 llalign
 440             } else {
 441                 C_uint(ccx, type_of::align_of(ccx, tp_ty))
 442             }
 443         }
 444         (_, "pref_align_of") => {
 445             let tp_ty = *substs.types.get(FnSpace, 0);
 446             let lltp_ty = type_of::type_of(ccx, tp_ty);
 447             C_uint(ccx, machine::llalign_of_pref(ccx, lltp_ty))
 448         }
 449         (_, "drop_in_place") => {
 450             let tp_ty = *substs.types.get(FnSpace, 0);
 451             let ptr = if type_is_sized(tcx, tp_ty) {
 452                 llargs[0]
 453             } else {
 454                 let scratch = rvalue_scratch_datum(bcx, tp_ty, "tmp");
 455                 Store(bcx, llargs[0], expr::get_dataptr(bcx, scratch.val));
 456                 Store(bcx, llargs[1], expr::get_meta(bcx, scratch.val));
 457                 fcx.schedule_lifetime_end(cleanup::CustomScope(cleanup_scope), scratch.val);
 458                 scratch.val
 459             };
 460             glue::drop_ty(bcx, ptr, tp_ty, call_debug_location);
 461             C_nil(ccx)
 462         }
 463         (_, "type_name") => {
 464             let tp_ty = *substs.types.get(FnSpace, 0);
 465             let ty_name = token::intern_and_get_ident(&tp_ty.to_string());
 466             C_str_slice(ccx, ty_name)
 467         }
 468         (_, "type_id") => {
 469             let hash = ccx.tcx().hash_crate_independent(*substs.types.get(FnSpace, 0),
 470                                                         &ccx.link_meta().crate_hash);
 471             C_u64(ccx, hash)
 472         }
 473         (_, "init_dropped") => {
 474             let tp_ty = *substs.types.get(FnSpace, 0);
 475             if !return_type_is_void(ccx, tp_ty) {
 476                 drop_done_fill_mem(bcx, llresult, tp_ty);
 477             }
 478             C_nil(ccx)
 479         }
 480         (_, "init") => {
 481             let tp_ty = *substs.types.get(FnSpace, 0);
 482             if !return_type_is_void(ccx, tp_ty) {
 483                 // Just zero out the stack slot. (See comment on base::memzero for explanation)
 484                 init_zero_mem(bcx, llresult, tp_ty);
 485             }
 486             C_nil(ccx)
 487         }
 488         // Effectively no-ops
 489         (_, "uninit") | (_, "forget") => {
 490             C_nil(ccx)
 491         }
 492         (_, "needs_drop") => {
 493             let tp_ty = *substs.types.get(FnSpace, 0);
 494
 495             C_bool(ccx, bcx.fcx.type_needs_drop(tp_ty))
 496         }
 497         (_, "offset") => {
 498             let ptr = llargs[0];
 499             let offset = llargs[1];
 500             InBoundsGEP(bcx, ptr, &[offset])
 501         }
 502         (_, "arith_offset") => {
 503             let ptr = llargs[0];
 504             let offset = llargs[1];
 505             GEP(bcx, ptr, &[offset])
 506         }
 507
 508         (_, "copy_nonoverlapping") => {
 509             copy_intrinsic(bcx,
 510                            false,
 511                            false,
 512                            *substs.types.get(FnSpace, 0),
 513                            llargs[1],
 514                            llargs[0],
 515                            llargs[2],
 516                            call_debug_location)
 517         }
 518         (_, "copy") => {
 519             copy_intrinsic(bcx,
 520                            true,
 521                            false,
 522                            *substs.types.get(FnSpace, 0),
 523                            llargs[1],
 524                            llargs[0],
 525                            llargs[2],
 526                            call_debug_location)
 527         }
 528         (_, "write_bytes") => {
 529             memset_intrinsic(bcx,
 530                              false,
 531                              *substs.types.get(FnSpace, 0),
 532                              llargs[0],
 533                              llargs[1],
 534                              llargs[2],
 535                              call_debug_location)
 536         }
 537
 538         (_, "volatile_copy_nonoverlapping_memory") => {
 539             copy_intrinsic(bcx,
 540                            false,
 541                            true,
 542                            *substs.types.get(FnSpace, 0),
 543                            llargs[0],
 544                            llargs[1],
 545                            llargs[2],
 546                            call_debug_location)
 547         }
 548         (_, "volatile_copy_memory") => {
 549             copy_intrinsic(bcx,
 550                            true,
 551                            true,
 552                            *substs.types.get(FnSpace, 0),
 553                            llargs[0],
 554                            llargs[1],
 555                            llargs[2],
 556                            call_debug_location)
 557         }
 558         (_, "volatile_set_memory") => {
 559             memset_intrinsic(bcx,
 560                              true,
 561                              *substs.types.get(FnSpace, 0),
 562                              llargs[0],
 563                              llargs[1],
 564                              llargs[2],
 565                              call_debug_location)
 566         }
 567         (_, "volatile_load") => {
 568             let tp_ty = *substs.types.get(FnSpace, 0);
 569             let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty);
 570             let load = VolatileLoad(bcx, ptr);
 571             unsafe {
 572                 llvm::LLVMSetAlignment(load, type_of::align_of(ccx, tp_ty));
 573             }
 574             to_arg_ty(bcx, load, tp_ty)
 575         },
 576         (_, "volatile_store") => {
 577             let tp_ty = *substs.types.get(FnSpace, 0);
 578             let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty);
 579             let val = from_arg_ty(bcx, llargs[1], tp_ty);
 580             let store = VolatileStore(bcx, val, ptr);
 581             unsafe {
 582                 llvm::LLVMSetAlignment(store, type_of::align_of(ccx, tp_ty));
 583             }
 584             C_nil(ccx)
 585         },
 586
 587         (_, "ctlz8") => count_zeros_intrinsic(bcx,
 588                                               "llvm.ctlz.i8",
 589                                               llargs[0],
 590                                               call_debug_location),
 591         (_, "ctlz16") => count_zeros_intrinsic(bcx,
 592                                                "llvm.ctlz.i16",
 593                                                llargs[0],
 594                                                call_debug_location),
 595         (_, "ctlz32") => count_zeros_intrinsic(bcx,
 596                                                "llvm.ctlz.i32",
 597                                                llargs[0],
 598                                                call_debug_location),
 599         (_, "ctlz64") => count_zeros_intrinsic(bcx,
 600                                                "llvm.ctlz.i64",
 601                                                llargs[0],
 602                                                call_debug_location),
 603         (_, "cttz8") => count_zeros_intrinsic(bcx,
 604                                               "llvm.cttz.i8",
 605                                               llargs[0],
 606                                               call_debug_location),
 607         (_, "cttz16") => count_zeros_intrinsic(bcx,
 608                                                "llvm.cttz.i16",
 609                                                llargs[0],
 610                                                call_debug_location),
 611         (_, "cttz32") => count_zeros_intrinsic(bcx,
 612                                                "llvm.cttz.i32",
 613                                                llargs[0],
 614                                                call_debug_location),
 615         (_, "cttz64") => count_zeros_intrinsic(bcx,
 616                                                "llvm.cttz.i64",
 617                                                llargs[0],
 618                                                call_debug_location),
 619
 620         (_, "i8_add_with_overflow") =>
 621             with_overflow_intrinsic(bcx,
 622                                     "llvm.sadd.with.overflow.i8",
 623                                     llargs[0],
 624                                     llargs[1],
 625                                     llresult,
 626                                     call_debug_location),
 627         (_, "i16_add_with_overflow") =>
 628             with_overflow_intrinsic(bcx,
 629                                     "llvm.sadd.with.overflow.i16",
 630                                     llargs[0],
 631                                     llargs[1],
 632                                     llresult,
 633                                     call_debug_location),
 634         (_, "i32_add_with_overflow") =>
 635             with_overflow_intrinsic(bcx,
 636                                     "llvm.sadd.with.overflow.i32",
 637                                     llargs[0],
 638                                     llargs[1],
 639                                     llresult,
 640                                     call_debug_location),
 641         (_, "i64_add_with_overflow") =>
 642             with_overflow_intrinsic(bcx,
 643                                     "llvm.sadd.with.overflow.i64",
 644                                     llargs[0],
 645                                     llargs[1],
 646                                     llresult,
 647                                     call_debug_location),
 648
 649         (_, "u8_add_with_overflow") =>
 650             with_overflow_intrinsic(bcx,
 651                                     "llvm.uadd.with.overflow.i8",
 652                                     llargs[0],
 653                                     llargs[1],
 654                                     llresult,
 655                                     call_debug_location),
 656         (_, "u16_add_with_overflow") =>
 657             with_overflow_intrinsic(bcx,
 658                                     "llvm.uadd.with.overflow.i16",
 659                                     llargs[0],
 660                                     llargs[1],
 661                                     llresult,
 662                                     call_debug_location),
 663         (_, "u32_add_with_overflow") =>
 664             with_overflow_intrinsic(bcx,
 665                                     "llvm.uadd.with.overflow.i32",
 666                                     llargs[0],
 667                                     llargs[1],
 668                                     llresult,
 669                                     call_debug_location),
 670         (_, "u64_add_with_overflow") =>
 671             with_overflow_intrinsic(bcx,
 672                                     "llvm.uadd.with.overflow.i64",
 673                                     llargs[0],
 674                                     llargs[1],
 675                                     llresult,
 676                                     call_debug_location),
 677         (_, "i8_sub_with_overflow") =>
 678             with_overflow_intrinsic(bcx,
 679                                     "llvm.ssub.with.overflow.i8",
 680                                     llargs[0],
 681                                     llargs[1],
 682                                     llresult,
 683                                     call_debug_location),
 684         (_, "i16_sub_with_overflow") =>
 685             with_overflow_intrinsic(bcx,
 686                                     "llvm.ssub.with.overflow.i16",
 687                                     llargs[0],
 688                                     llargs[1],
 689                                     llresult,
 690                                     call_debug_location),
 691         (_, "i32_sub_with_overflow") =>
 692             with_overflow_intrinsic(bcx,
 693                                     "llvm.ssub.with.overflow.i32",
 694                                     llargs[0],
 695                                     llargs[1],
 696                                     llresult,
 697                                     call_debug_location),
 698         (_, "i64_sub_with_overflow") =>
 699             with_overflow_intrinsic(bcx,
 700                                     "llvm.ssub.with.overflow.i64",
 701                                     llargs[0],
 702                                     llargs[1],
 703                                     llresult,
 704                                     call_debug_location),
 705         (_, "u8_sub_with_overflow") =>
 706             with_overflow_intrinsic(bcx,
 707                                     "llvm.usub.with.overflow.i8",
 708                                     llargs[0],
 709                                     llargs[1],
 710                                     llresult,
 711                                     call_debug_location),
 712         (_, "u16_sub_with_overflow") =>
 713             with_overflow_intrinsic(bcx,
 714                                     "llvm.usub.with.overflow.i16",
 715                                     llargs[0],
 716                                     llargs[1],
 717                                     llresult,
 718                                     call_debug_location),
 719         (_, "u32_sub_with_overflow") =>
 720             with_overflow_intrinsic(bcx,
 721                                     "llvm.usub.with.overflow.i32",
 722                                     llargs[0],
 723                                     llargs[1],
 724                                     llresult,
 725                                     call_debug_location),
 726         (_, "u64_sub_with_overflow") =>
 727             with_overflow_intrinsic(bcx,
 728                                     "llvm.usub.with.overflow.i64",
 729                                     llargs[0],
 730                                     llargs[1],
 731                                     llresult,
 732                                     call_debug_location),
 733         (_, "i8_mul_with_overflow") =>
 734             with_overflow_intrinsic(bcx,
 735                                     "llvm.smul.with.overflow.i8",
 736                                     llargs[0],
 737                                     llargs[1],
 738                                     llresult,
 739                                     call_debug_location),
 740         (_, "i16_mul_with_overflow") =>
 741             with_overflow_intrinsic(bcx,
 742                                     "llvm.smul.with.overflow.i16",
 743                                     llargs[0],
 744                                     llargs[1],
 745                                     llresult,
 746                                     call_debug_location),
 747         (_, "i32_mul_with_overflow") =>
 748             with_overflow_intrinsic(bcx,
 749                                     "llvm.smul.with.overflow.i32",
 750                                     llargs[0],
 751                                     llargs[1],
 752                                     llresult,
 753                                     call_debug_location),
 754         (_, "i64_mul_with_overflow") =>
 755             with_overflow_intrinsic(bcx,
 756                                     "llvm.smul.with.overflow.i64",
 757                                     llargs[0],
 758                                     llargs[1],
 759                                     llresult,
 760                                     call_debug_location),
 761         (_, "u8_mul_with_overflow") =>
 762             with_overflow_intrinsic(bcx,
 763                                     "llvm.umul.with.overflow.i8",
 764                                     llargs[0],
 765                                     llargs[1],
 766                                     llresult,
 767                                     call_debug_location),
 768         (_, "u16_mul_with_overflow") =>
 769             with_overflow_intrinsic(bcx,
 770                                     "llvm.umul.with.overflow.i16",
 771                                     llargs[0],
 772                                     llargs[1],
 773                                     llresult,
 774                                     call_debug_location),
 775         (_, "u32_mul_with_overflow") =>
 776             with_overflow_intrinsic(bcx,
 777                                     "llvm.umul.with.overflow.i32",
 778                                     llargs[0],
 779                                     llargs[1],
 780                                     llresult,
 781                                     call_debug_location),
 782         (_, "u64_mul_with_overflow") =>
 783             with_overflow_intrinsic(bcx,
 784                                     "llvm.umul.with.overflow.i64",
 785                                     llargs[0],
 786                                     llargs[1],
 787                                     llresult,
 788                                     call_debug_location),
 789
 790         (_, "unchecked_udiv") => UDiv(bcx, llargs[0], llargs[1], call_debug_location),
 791         (_, "unchecked_sdiv") => SDiv(bcx, llargs[0], llargs[1], call_debug_location),
 792         (_, "unchecked_urem") => URem(bcx, llargs[0], llargs[1], call_debug_location),
 793         (_, "unchecked_srem") => SRem(bcx, llargs[0], llargs[1], call_debug_location),
 794
 795         (_, "overflowing_add") => Add(bcx, llargs[0], llargs[1], call_debug_location),
 796         (_, "overflowing_sub") => Sub(bcx, llargs[0], llargs[1], call_debug_location),
 797         (_, "overflowing_mul") => Mul(bcx, llargs[0], llargs[1], call_debug_location),
 798
 799         (_, "return_address") => {
 800             if !fcx.caller_expects_out_pointer {
 801                 tcx.sess.span_err(call_info.span,
 802                                   "invalid use of `return_address` intrinsic: function \
 803                                    does not use out pointer");
 804                 C_null(Type::i8p(ccx))
 805             } else {
 806                 PointerCast(bcx, llvm::get_param(fcx.llfn, 0), Type::i8p(ccx))
 807             }
 808         }
 809
 810         (_, "discriminant_value") => {
 811             let val_ty = substs.types.get(FnSpace, 0);
 812             match val_ty.sty {
 813                 ty::TyEnum(..) => {
 814                     let repr = adt::represent_type(ccx, *val_ty);
 815                     adt::trans_get_discr(bcx, &*repr, llargs[0], Some(llret_ty))
 816                 }
 817                 _ => C_null(llret_ty)
 818             }
 819         }
 820         (_, name) if name.starts_with("simd_") => {
 821             generic_simd_intrinsic(bcx, name,
 822                                    substs,
 823                                    callee_ty,
 824                                    expr_arguments,
 825                                    &llargs,
 826                                    ret_ty, llret_ty,
 827                                    call_debug_location,
 828                                    call_info)
 829         }
 830         // This requires that atomic intrinsics follow a specific naming pattern:
 831         // "atomic_<operation>[_<ordering>]", and no ordering means SeqCst
 832         (_, name) if name.starts_with("atomic_") => {
 833             let split: Vec<&str> = name.split('_').collect();
 834             assert!(split.len() >= 2, "Atomic intrinsic not correct format");
 835
 836             let order = if split.len() == 2 {
 837                 llvm::SequentiallyConsistent
 838             } else {
 839                 match split[2] {
 840                     "unordered" => llvm::Unordered,
 841                     "relaxed" => llvm::Monotonic,
 842                     "acq"     => llvm::Acquire,
 843                     "rel"     => llvm::Release,
 844                     "acqrel"  => llvm::AcquireRelease,
 845                     _ => ccx.sess().fatal("unknown ordering in atomic intrinsic")
 846                 }
 847             };
 848
 849             match split[1] {
 850                 "cxchg" => {
 851                     // See include/llvm/IR/Instructions.h for their implementation
 852                     // of this, I assume that it's good enough for us to use for
 853                     // now.
 854                     let strongest_failure_ordering = match order {
 855                         llvm::NotAtomic | llvm::Unordered =>
 856                             ccx.sess().fatal("cmpxchg must be atomic"),
 857
 858                         llvm::Monotonic | llvm::Release =>
 859                             llvm::Monotonic,
 860
 861                         llvm::Acquire | llvm::AcquireRelease =>
 862                             llvm::Acquire,
 863
 864                         llvm::SequentiallyConsistent =>
 865                             llvm::SequentiallyConsistent
 866                     };
 867
 868                     let tp_ty = *substs.types.get(FnSpace, 0);
 869                     let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty);
 870                     let cmp = from_arg_ty(bcx, llargs[1], tp_ty);
 871                     let src = from_arg_ty(bcx, llargs[2], tp_ty);
 872                     let res = AtomicCmpXchg(bcx, ptr, cmp, src, order,
 873                                             strongest_failure_ordering);
 874                     ExtractValue(bcx, res, 0)
 875                 }
 876
 877                 "load" => {
 878                     let tp_ty = *substs.types.get(FnSpace, 0);
 879                     let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty);
 880                     to_arg_ty(bcx, AtomicLoad(bcx, ptr, order), tp_ty)
 881                 }
 882                 "store" => {
 883                     let tp_ty = *substs.types.get(FnSpace, 0);
 884                     let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty);
 885                     let val = from_arg_ty(bcx, llargs[1], tp_ty);
 886                     AtomicStore(bcx, val, ptr, order);
 887                     C_nil(ccx)
 888                 }
 889
 890                 "fence" => {
 891                     AtomicFence(bcx, order, llvm::CrossThread);
 892                     C_nil(ccx)
 893                 }
 894
 895                 "singlethreadfence" => {
 896                     AtomicFence(bcx, order, llvm::SingleThread);
 897                     C_nil(ccx)
 898                 }
 899
 900                 // These are all AtomicRMW ops
 901                 op => {
 902                     let atom_op = match op {
 903                         "xchg"  => llvm::AtomicXchg,
 904                         "xadd"  => llvm::AtomicAdd,
 905                         "xsub"  => llvm::AtomicSub,
 906                         "and"   => llvm::AtomicAnd,
 907                         "nand"  => llvm::AtomicNand,
 908                         "or"    => llvm::AtomicOr,
 909                         "xor"   => llvm::AtomicXor,
 910                         "max"   => llvm::AtomicMax,
 911                         "min"   => llvm::AtomicMin,
 912                         "umax"  => llvm::AtomicUMax,
 913                         "umin"  => llvm::AtomicUMin,
 914                         _ => ccx.sess().fatal("unknown atomic operation")
 915                     };
 916
 917                     let tp_ty = *substs.types.get(FnSpace, 0);
 918                     let ptr = to_arg_ty_ptr(bcx, llargs[0], tp_ty);
 919                     let val = from_arg_ty(bcx, llargs[1], tp_ty);
 920                     AtomicRMW(bcx, atom_op, ptr, val, order)
 921                 }
 922             }
 923
 924         }
 925
 926         (_, _) => {
 927             let intr = match Intrinsic::find(tcx, &name) {
 928                 Some(intr) => intr,
 929                 None => ccx.sess().span_bug(foreign_item.span, "unknown intrinsic"),
 930             };
 931             fn one<T>(x: Vec<T>) -> T {
 932                 assert_eq!(x.len(), 1);
 933                 x.into_iter().next().unwrap()
 934             }
 935             fn ty_to_type(ccx: &CrateContext, t: &intrinsics::Type,
 936                           any_changes_needed: &mut bool) -> Vec<Type> {
 937                 use intrinsics::Type::*;
 938                 match *t {
 939                     Void => vec![Type::void(ccx)],
 940                     Integer(_signed, width, llvm_width) => {
 941                         *any_changes_needed |= width != llvm_width;
 942                         vec![Type::ix(ccx, llvm_width as u64)]
 943                     }
 944                     Float(x) => {
 945                         match x {
 946                             32 => vec![Type::f32(ccx)],
 947                             64 => vec![Type::f64(ccx)],
 948                             _ => unreachable!()
 949                         }
 950                     }
 951                     Pointer(ref t, ref llvm_elem, _const) => {
 952                         *any_changes_needed |= llvm_elem.is_some();
 953
 954                         let t = llvm_elem.as_ref().unwrap_or(t);
 955                         let elem = one(ty_to_type(ccx, t,
 956                                                   any_changes_needed));
 957                         vec![elem.ptr_to()]
 958                     }
 959                     Vector(ref t, ref llvm_elem, length) => {
 960                         *any_changes_needed |= llvm_elem.is_some();
 961
 962                         let t = llvm_elem.as_ref().unwrap_or(t);
 963                         let elem = one(ty_to_type(ccx, t,
 964                                                   any_changes_needed));
 965                         vec![Type::vector(&elem,
 966                                           length as u64)]
 967                     }
 968                     Aggregate(false, ref contents) => {
 969                         let elems = contents.iter()
 970                                             .map(|t| one(ty_to_type(ccx, t, any_changes_needed)))
 971                                             .collect::<Vec<_>>();
 972                         vec![Type::struct_(ccx, &elems, false)]
 973                     }
 974                     Aggregate(true, ref contents) => {
 975                         *any_changes_needed = true;
 976                         contents.iter()
 977                                 .flat_map(|t| ty_to_type(ccx, t, any_changes_needed))
 978                                 .collect()
 979                     }
 980                 }
 981             }
 982
 983             // This allows an argument list like `foo, (bar, baz),
 984             // qux` to be converted into `foo, bar, baz, qux`, integer
 985             // arguments to be truncated as needed and pointers to be
 986             // cast.
 987             fn modify_as_needed<'blk, 'tcx>(bcx: Block<'blk, 'tcx>,
 988                                             t: &intrinsics::Type,
 989                                             arg_type: Ty<'tcx>,
 990                                             llarg: ValueRef)
 991                                             -> Vec<ValueRef>
 992             {
 993                 match *t {
 994                     intrinsics::Type::Aggregate(true, ref contents) => {
 995                         // We found a tuple that needs squishing! So
 996                         // run over the tuple and load each field.
 997                         //
 998                         // This assumes the type is "simple", i.e. no
 999                         // destructors, and the contents are SIMD
1000                         // etc.
1001                         assert!(!bcx.fcx.type_needs_drop(arg_type));
1002
1003                         let repr = adt::represent_type(bcx.ccx(), arg_type);
1004                         let repr_ptr = &*repr;
1005                         (0..contents.len())
1006                             .map(|i| {
1007                                 Load(bcx, adt::trans_field_ptr(bcx, repr_ptr, llarg, 0, i))
1008                             })
1009                             .collect()
1010                     }
1011                     intrinsics::Type::Pointer(_, Some(ref llvm_elem), _) => {
1012                         let llvm_elem = one(ty_to_type(bcx.ccx(), llvm_elem, &mut false));
1013                         vec![PointerCast(bcx, llarg,
1014                                          llvm_elem.ptr_to())]
1015                     }
1016                     intrinsics::Type::Vector(_, Some(ref llvm_elem), length) => {
1017                         let llvm_elem = one(ty_to_type(bcx.ccx(), llvm_elem, &mut false));
1018                         vec![BitCast(bcx, llarg,
1019                                      Type::vector(&llvm_elem, length as u64))]
1020                     }
1021                     intrinsics::Type::Integer(_, width, llvm_width) if width != llvm_width => {
1022                         // the LLVM intrinsic uses a smaller integer
1023                         // size than the C intrinsic's signature, so
1024                         // we have to trim it down here.
1025                         vec![Trunc(bcx, llarg, Type::ix(bcx.ccx(), llvm_width as u64))]
1026                     }
1027                     _ => vec![llarg],
1028                 }
1029             }
1030
1031
1032             let mut any_changes_needed = false;
1033             let inputs = intr.inputs.iter()
1034                                     .flat_map(|t| ty_to_type(ccx, t, &mut any_changes_needed))
1035                                     .collect::<Vec<_>>();
1036
1037             let mut out_changes = false;
1038             let outputs = one(ty_to_type(ccx, &intr.output, &mut out_changes));
1039             // outputting a flattened aggregate is nonsense
1040             assert!(!out_changes);
1041
1042             let llargs = if !any_changes_needed {
1043                 // no aggregates to flatten, so no change needed
1044                 llargs
1045             } else {
1046                 // there are some aggregates that need to be flattened
1047                 // in the LLVM call, so we need to run over the types
1048                 // again to find them and extract the arguments
1049                 intr.inputs.iter()
1050                            .zip(&llargs)
1051                            .zip(&arg_tys)
1052                            .flat_map(|((t, llarg), ty)| modify_as_needed(bcx, t, ty, *llarg))
1053                            .collect()
1054             };
1055             assert_eq!(inputs.len(), llargs.len());
1056
1057             let val = match intr.definition {
1058                 intrinsics::IntrinsicDef::Named(name) => {
1059                     let f = declare::declare_cfn(ccx,
1060                                                  name,
1061                                                  Type::func(&inputs, &outputs),
1062                                                  tcx.mk_nil());
1063                     Call(bcx, f, &llargs, None, call_debug_location)
1064                 }
1065             };
1066
1067             match intr.output {
1068                 intrinsics::Type::Aggregate(flatten, ref elems) => {
1069                     // the output is a tuple so we need to munge it properly
1070                     assert!(!flatten);
1071
1072                     for i in 0..elems.len() {
1073                         let val = ExtractValue(bcx, val, i);
1074                         Store(bcx, val, StructGEP(bcx, llresult, i));
1075                     }
1076                     C_nil(ccx)
1077                 }
1078                 _ => val,
1079             }
1080         }
1081     };
1082
1083     if val_ty(llval) != Type::void(ccx) &&
1084        machine::llsize_of_alloc(ccx, val_ty(llval)) != 0 {
1085         store_ty(bcx, llval, llresult, ret_ty);
1086     }
1087
1088     // If we made a temporary stack slot, let's clean it up
1089     match dest {
1090         expr::Ignore => {
1091             bcx = glue::drop_ty(bcx, llresult, ret_ty, call_debug_location);
1092             call_lifetime_end(bcx, llresult);
1093         }
1094         expr::SaveIn(_) => {}
1095     }
1096
1097     fcx.pop_and_trans_custom_cleanup_scope(bcx, cleanup_scope);
1098
1099     Result::new(bcx, llresult)
1100 }
1101
1102 fn copy_intrinsic<'blk, 'tcx>(bcx: Block<'blk, 'tcx>,
1103                               allow_overlap: bool,
1104                               volatile: bool,
1105                               tp_ty: Ty<'tcx>,
1106                               dst: ValueRef,
1107                               src: ValueRef,
1108                               count: ValueRef,
1109                               call_debug_location: DebugLoc)
1110                               -> ValueRef {
1111     let ccx = bcx.ccx();
1112     let lltp_ty = type_of::type_of(ccx, tp_ty);
1113     let align = C_i32(ccx, type_of::align_of(ccx, tp_ty) as i32);
1114     let size = machine::llsize_of(ccx, lltp_ty);
1115     let int_size = machine::llbitsize_of_real(ccx, ccx.int_type());
1116
1117     let operation = if allow_overlap {
1118         "memmove"
1119     } else {
1120         "memcpy"
1121     };
1122
1123     let name = format!("llvm.{}.p0i8.p0i8.i{}", operation, int_size);
1124
1125     let dst_ptr = PointerCast(bcx, dst, Type::i8p(ccx));
1126     let src_ptr = PointerCast(bcx, src, Type::i8p(ccx));
1127     let llfn = ccx.get_intrinsic(&name);
1128
1129     Call(bcx,
1130          llfn,
1131          &[dst_ptr,
1132            src_ptr,
1133            Mul(bcx, size, count, DebugLoc::None),
1134            align,
1135            C_bool(ccx, volatile)],
1136          None,
1137          call_debug_location)
1138 }
1139
1140 fn memset_intrinsic<'blk, 'tcx>(bcx: Block<'blk, 'tcx>,
1141                                 volatile: bool,
1142                                 tp_ty: Ty<'tcx>,
1143                                 dst: ValueRef,
1144                                 val: ValueRef,
1145                                 count: ValueRef,
1146                                 call_debug_location: DebugLoc)
1147                                 -> ValueRef {
1148     let ccx = bcx.ccx();
1149     let lltp_ty = type_of::type_of(ccx, tp_ty);
1150     let align = C_i32(ccx, type_of::align_of(ccx, tp_ty) as i32);
1151     let size = machine::llsize_of(ccx, lltp_ty);
1152     let int_size = machine::llbitsize_of_real(ccx, ccx.int_type());
1153
1154     let name = format!("llvm.memset.p0i8.i{}", int_size);
1155
1156     let dst_ptr = PointerCast(bcx, dst, Type::i8p(ccx));
1157     let llfn = ccx.get_intrinsic(&name);
1158
1159     Call(bcx,
1160          llfn,
1161          &[dst_ptr,
1162            val,
1163            Mul(bcx, size, count, DebugLoc::None),
1164            align,
1165            C_bool(ccx, volatile)],
1166          None,
1167          call_debug_location)
1168 }
1169
1170 fn count_zeros_intrinsic(bcx: Block,
1171                          name: &'static str,
1172                          val: ValueRef,
1173                          call_debug_location: DebugLoc)
1174                          -> ValueRef {
1175     let y = C_bool(bcx.ccx(), false);
1176     let llfn = bcx.ccx().get_intrinsic(&name);
1177     Call(bcx, llfn, &[val, y], None, call_debug_location)
1178 }
1179
1180 fn with_overflow_intrinsic<'blk, 'tcx>(bcx: Block<'blk, 'tcx>,
1181                                        name: &'static str,
1182                                        a: ValueRef,
1183                                        b: ValueRef,
1184                                        out: ValueRef,
1185                                        call_debug_location: DebugLoc)
1186                                        -> ValueRef {
1187     let llfn = bcx.ccx().get_intrinsic(&name);
1188
1189     // Convert `i1` to a `bool`, and write it to the out parameter
1190     let val = Call(bcx, llfn, &[a, b], None, call_debug_location);
1191     let result = ExtractValue(bcx, val, 0);
1192     let overflow = ZExt(bcx, ExtractValue(bcx, val, 1), Type::bool(bcx.ccx()));
1193     Store(bcx, result, StructGEP(bcx, out, 0));
1194     Store(bcx, overflow, StructGEP(bcx, out, 1));
1195
1196     C_nil(bcx.ccx())
1197 }
1198
1199 fn try_intrinsic<'blk, 'tcx>(bcx: Block<'blk, 'tcx>,
1200                              func: ValueRef,
1201                              data: ValueRef,
1202                              dest: ValueRef,
1203                              dloc: DebugLoc) -> Block<'blk, 'tcx> {
1204     if bcx.sess().no_landing_pads() {
1205         Call(bcx, func, &[data], None, dloc);
1206         Store(bcx, C_null(Type::i8p(bcx.ccx())), dest);
1207         bcx
1208     } else if wants_msvc_seh(bcx.sess()) {
1209         trans_msvc_try(bcx, func, data, dest, dloc)
1210     } else {
1211         trans_gnu_try(bcx, func, data, dest, dloc)
1212     }
1213 }
1214
1215 // MSVC's definition of the `rust_try` function. The exact implementation here
1216 // is a little different than the GNU (standard) version below, not only because
1217 // of the personality function but also because of the other fiddly bits about
1218 // SEH. LLVM also currently requires us to structure this a very particular way
1219 // as explained below.
1220 //
1221 // Like with the GNU version we generate a shim wrapper
1222 fn trans_msvc_try<'blk, 'tcx>(bcx: Block<'blk, 'tcx>,
1223                               func: ValueRef,
1224                               data: ValueRef,
1225                               dest: ValueRef,
1226                               dloc: DebugLoc) -> Block<'blk, 'tcx> {
1227     let llfn = get_rust_try_fn(bcx.fcx, &mut |try_fn_ty, output| {
1228         let ccx = bcx.ccx();
1229         let dloc = DebugLoc::None;
1230         let rust_try = declare::define_internal_rust_fn(ccx, "__rust_try",
1231                                                          try_fn_ty);
1232         let (fcx, block_arena);
1233         block_arena = TypedArena::new();
1234         fcx = new_fn_ctxt(ccx, rust_try, ast::DUMMY_NODE_ID, false,
1235                           output, ccx.tcx().mk_substs(Substs::trans_empty()),
1236                           None, &block_arena);
1237         let bcx = init_function(&fcx, true, output);
1238         let then = fcx.new_temp_block("then");
1239         let catch = fcx.new_temp_block("catch");
1240         let catch_return = fcx.new_temp_block("catch-return");
1241         let catch_resume = fcx.new_temp_block("catch-resume");
1242         let personality = fcx.eh_personality();
1243
1244         let eh_typeid_for = ccx.get_intrinsic(&"llvm.eh.typeid.for");
1245         let rust_try_filter = match bcx.tcx().lang_items.msvc_try_filter() {
1246             Some(did) => callee::trans_fn_ref(ccx, did, ExprId(0),
1247                                               bcx.fcx.param_substs).val,
1248             None => bcx.sess().bug("msvc_try_filter not defined"),
1249         };
1250
1251         // Type indicator for the exception being thrown, not entirely sure
1252         // what's going on here but it's what all the examples in LLVM use.
1253         let lpad_ty = Type::struct_(ccx, &[Type::i8p(ccx), Type::i32(ccx)],
1254                                     false);
1255
1256         llvm::SetFunctionAttribute(rust_try, llvm::Attribute::NoInline);
1257         llvm::SetFunctionAttribute(rust_try, llvm::Attribute::OptimizeNone);
1258         let func = llvm::get_param(rust_try, 0);
1259         let data = llvm::get_param(rust_try, 1);
1260
1261         // Invoke the function, specifying our two temporary landing pads as the
1262         // ext point. After the invoke we've terminated our basic block.
1263         Invoke(bcx, func, &[data], then.llbb, catch.llbb, None, dloc);
1264
1265         // All the magic happens in this landing pad, and this is basically the
1266         // only landing pad in rust tagged with "catch" to indicate that we're
1267         // catching an exception. The other catch handlers in the GNU version
1268         // below just catch *all* exceptions, but that's because most exceptions
1269         // are already filtered out by the gnu personality function.
1270         //
1271         // For MSVC we're just using a standard personality function that we
1272         // can't customize (e.g. _except_handler3 or __C_specific_handler), so
1273         // we need to do the exception filtering ourselves. This is currently
1274         // performed by the `__rust_try_filter` function. This function,
1275         // specified in the landingpad instruction, will be invoked by Windows
1276         // SEH routines and will return whether the exception in question can be
1277         // caught (aka the Rust runtime is the one that threw the exception).
1278         //
1279         // To get this to compile (currently LLVM segfaults if it's not in this
1280         // particular structure), when the landingpad is executing we test to
1281         // make sure that the ID of the exception being thrown is indeed the one
1282         // that we were expecting. If it's not, we resume the exception, and
1283         // otherwise we return the pointer that we got Full disclosure: It's not
1284         // clear to me what this `llvm.eh.typeid` stuff is doing *other* then
1285         // just allowing LLVM to compile this file without segfaulting. I would
1286         // expect the entire landing pad to just be:
1287         //
1288         //     %vals = landingpad ...
1289         //     %ehptr = extractvalue { i8*, i32 } %vals, 0
1290         //     ret i8* %ehptr
1291         //
1292         // but apparently LLVM chokes on this, so we do the more complicated
1293         // thing to placate it.
1294         let vals = LandingPad(catch, lpad_ty, personality, 1);
1295         let rust_try_filter = BitCast(catch, rust_try_filter, Type::i8p(ccx));
1296         AddClause(catch, vals, rust_try_filter);
1297         let ehptr = ExtractValue(catch, vals, 0);
1298         let sel = ExtractValue(catch, vals, 1);
1299         let filter_sel = Call(catch, eh_typeid_for, &[rust_try_filter], None,
1300                               dloc);
1301         let is_filter = ICmp(catch, llvm::IntEQ, sel, filter_sel, dloc);
1302         CondBr(catch, is_filter, catch_return.llbb, catch_resume.llbb, dloc);
1303
1304         // Our "catch-return" basic block is where we've determined that we
1305         // actually need to catch this exception, in which case we just return
1306         // the exception pointer.
1307         Ret(catch_return, ehptr, dloc);
1308
1309         // The "catch-resume" block is where we're running this landing pad but
1310         // we actually need to not catch the exception, so just resume the
1311         // exception to return.
1312         Resume(catch_resume, vals);
1313
1314         // On the successful branch we just return null.
1315         Ret(then, C_null(Type::i8p(ccx)), dloc);
1316
1317         return rust_try
1318     });
1319
1320     // Note that no invoke is used here because by definition this function
1321     // can't panic (that's what it's catching).
1322     let ret = Call(bcx, llfn, &[func, data], None, dloc);
1323     Store(bcx, ret, dest);
1324     return bcx;
1325 }
1326
1327 // Definition of the standard "try" function for Rust using the GNU-like model
1328 // of exceptions (e.g. the normal semantics of LLVM's landingpad and invoke
1329 // instructions).
1330 //
1331 // This translation is a little surprising because
1332 // we always call a shim function instead of inlining the call to `invoke`
1333 // manually here. This is done because in LLVM we're only allowed to have one
1334 // personality per function definition. The call to the `try` intrinsic is
1335 // being inlined into the function calling it, and that function may already
1336 // have other personality functions in play. By calling a shim we're
1337 // guaranteed that our shim will have the right personality function.
1338 //
1339 fn trans_gnu_try<'blk, 'tcx>(bcx: Block<'blk, 'tcx>,
1340                              func: ValueRef,
1341                              data: ValueRef,
1342                              dest: ValueRef,
1343                              dloc: DebugLoc) -> Block<'blk, 'tcx> {
1344     let llfn = get_rust_try_fn(bcx.fcx, &mut |try_fn_ty, output| {
1345         let ccx = bcx.ccx();
1346         let dloc = DebugLoc::None;
1347
1348         // Translates the shims described above:
1349         //
1350         //   bcx:
1351         //      invoke %func(%args...) normal %normal unwind %catch
1352         //
1353         //   normal:
1354         //      ret null
1355         //
1356         //   catch:
1357         //      (ptr, _) = landingpad
1358         //      ret ptr
1359
1360         let rust_try = declare::define_internal_rust_fn(ccx, "__rust_try", try_fn_ty);
1361         attributes::emit_uwtable(rust_try, true);
1362         let catch_pers = match bcx.tcx().lang_items.eh_personality_catch() {
1363             Some(did) => callee::trans_fn_ref(ccx, did, ExprId(0),
1364                                               bcx.fcx.param_substs).val,
1365             None => bcx.tcx().sess.bug("eh_personality_catch not defined"),
1366         };
1367
1368         let (fcx, block_arena);
1369         block_arena = TypedArena::new();
1370         fcx = new_fn_ctxt(ccx, rust_try, ast::DUMMY_NODE_ID, false,
1371                           output, ccx.tcx().mk_substs(Substs::trans_empty()),
1372                           None, &block_arena);
1373         let bcx = init_function(&fcx, true, output);
1374         let then = bcx.fcx.new_temp_block("then");
1375         let catch = bcx.fcx.new_temp_block("catch");
1376
1377         let func = llvm::get_param(rust_try, 0);
1378         let data = llvm::get_param(rust_try, 1);
1379         Invoke(bcx, func, &[data], then.llbb, catch.llbb, None, dloc);
1380         Ret(then, C_null(Type::i8p(ccx)), dloc);
1381
1382         // Type indicator for the exception being thrown.
1383         // The first value in this tuple is a pointer to the exception object being thrown.
1384         // The second value is a "selector" indicating which of the landing pad clauses
1385         // the exception's type had been matched to.  rust_try ignores the selector.
1386         let lpad_ty = Type::struct_(ccx, &[Type::i8p(ccx), Type::i32(ccx)],
1387                                     false);
1388         let vals = LandingPad(catch, lpad_ty, catch_pers, 1);
1389         AddClause(catch, vals, C_null(Type::i8p(ccx)));
1390         let ptr = ExtractValue(catch, vals, 0);
1391         Ret(catch, ptr, dloc);
1392         fcx.cleanup();
1393
1394         return rust_try
1395     });
1396
1397     // Note that no invoke is used here because by definition this function
1398     // can't panic (that's what it's catching).
1399     let ret = Call(bcx, llfn, &[func, data], None, dloc);
1400     Store(bcx, ret, dest);
1401     return bcx;
1402 }
1403
1404 // Helper to generate the `Ty` associated with `rust_try`
1405 fn get_rust_try_fn<'a, 'tcx>(fcx: &FunctionContext<'a, 'tcx>,
1406                              f: &mut FnMut(Ty<'tcx>,
1407                                            ty::FnOutput<'tcx>) -> ValueRef)
1408                              -> ValueRef {
1409     let ccx = fcx.ccx;
1410     if let Some(llfn) = *ccx.rust_try_fn().borrow() {
1411         return llfn
1412     }
1413
1414     // Define the type up front for the signature of the rust_try function.
1415     let tcx = ccx.tcx();
1416     let i8p = tcx.mk_mut_ptr(tcx.types.i8);
1417     let fn_ty = tcx.mk_bare_fn(ty::BareFnTy {
1418         unsafety: hir::Unsafety::Unsafe,
1419         abi: abi::Rust,
1420         sig: ty::Binder(ty::FnSig {
1421             inputs: vec![i8p],
1422             output: ty::FnOutput::FnConverging(tcx.mk_nil()),
1423             variadic: false,
1424         }),
1425     });
1426     let fn_ty = tcx.mk_fn(None, fn_ty);
1427     let output = ty::FnOutput::FnConverging(i8p);
1428     let try_fn_ty  = tcx.mk_bare_fn(ty::BareFnTy {
1429         unsafety: hir::Unsafety::Unsafe,
1430         abi: abi::Rust,
1431         sig: ty::Binder(ty::FnSig {
1432             inputs: vec![fn_ty, i8p],
1433             output: output,
1434             variadic: false,
1435         }),
1436     });
1437     let rust_try = f(tcx.mk_fn(None, try_fn_ty), output);
1438     *ccx.rust_try_fn().borrow_mut() = Some(rust_try);
1439     return rust_try
1440 }
1441
1442 fn generic_simd_intrinsic<'blk, 'tcx, 'a>
1443     (bcx: Block<'blk, 'tcx>,
1444      name: &str,
1445      substs: subst::Substs<'tcx>,
1446      callee_ty: Ty<'tcx>,
1447      args: Option<&[P<hir::Expr>]>,
1448      llargs: &[ValueRef],
1449      ret_ty: Ty<'tcx>,
1450      llret_ty: Type,
1451      call_debug_location: DebugLoc,
1452      call_info: NodeIdAndSpan) -> ValueRef
1453 {
1454     // macros for error handling:
1455     macro_rules! emit_error {
1456         ($msg: tt) => {
1457             emit_error!($msg, )
1458         };
1459         ($msg: tt, $($fmt: tt)*) => {
1460             bcx.sess().span_err(call_info.span,
1461                                 &format!(concat!("invalid monomorphization of `{}` intrinsic: ",
1462                                                  $msg),
1463                                          name, $($fmt)*));
1464         }
1465     }
1466     macro_rules! require {
1467         ($cond: expr, $($fmt: tt)*) => {
1468             if !$cond {
1469                 emit_error!($($fmt)*);
1470                 return C_null(llret_ty)
1471             }
1472         }
1473     }
1474     macro_rules! require_simd {
1475         ($ty: expr, $position: expr) => {
1476             require!($ty.is_simd(), "expected SIMD {} type, found non-SIMD `{}`", $position, $ty)
1477         }
1478     }
1479
1480
1481
1482     let tcx = bcx.tcx();
1483     let arg_tys = match callee_ty.sty {
1484         ty::TyBareFn(_, ref f) => {
1485             bcx.tcx().erase_late_bound_regions(&f.sig.inputs())
1486         }
1487         _ => unreachable!()
1488     };
1489
1490     // every intrinsic takes a SIMD vector as its first argument
1491     require_simd!(arg_tys[0], "input");
1492     let in_ty = arg_tys[0];
1493     let in_elem = arg_tys[0].simd_type(tcx);
1494     let in_len = arg_tys[0].simd_size(tcx);
1495
1496     let comparison = match name {
1497         "simd_eq" => Some(hir::BiEq),
1498         "simd_ne" => Some(hir::BiNe),
1499         "simd_lt" => Some(hir::BiLt),
1500         "simd_le" => Some(hir::BiLe),
1501         "simd_gt" => Some(hir::BiGt),
1502         "simd_ge" => Some(hir::BiGe),
1503         _ => None
1504     };
1505
1506     if let Some(cmp_op) = comparison {
1507         require_simd!(ret_ty, "return");
1508
1509         let out_len = ret_ty.simd_size(tcx);
1510         require!(in_len == out_len,
1511                  "expected return type with length {} (same as input type `{}`), \
1512                   found `{}` with length {}",
1513                  in_len, in_ty,
1514                  ret_ty, out_len);
1515         require!(llret_ty.element_type().kind() == llvm::Integer,
1516                  "expected return type with integer elements, found `{}` with non-integer `{}`",
1517                  ret_ty,
1518                  ret_ty.simd_type(tcx));
1519
1520         return compare_simd_types(bcx,
1521                                   llargs[0],
1522                                   llargs[1],
1523                                   in_elem,
1524                                   llret_ty,
1525                                   cmp_op,
1526                                   call_debug_location)
1527     }
1528
1529     if name.starts_with("simd_shuffle") {
1530         let n: usize = match name["simd_shuffle".len()..].parse() {
1531             Ok(n) => n,
1532             Err(_) => tcx.sess.span_bug(call_info.span,
1533                                         "bad `simd_shuffle` instruction only caught in trans?")
1534         };
1535
1536         require_simd!(ret_ty, "return");
1537
1538         let out_len = ret_ty.simd_size(tcx);
1539         require!(out_len == n,
1540                  "expected return type of length {}, found `{}` with length {}",
1541                  n, ret_ty, out_len);
1542         require!(in_elem == ret_ty.simd_type(tcx),
1543                  "expected return element type `{}` (element of input `{}`), \
1544                   found `{}` with element type `{}`",
1545                  in_elem, in_ty,
1546                  ret_ty, ret_ty.simd_type(tcx));
1547
1548         let total_len = in_len as u64 * 2;
1549
1550         let vector = match args {
1551             Some(args) => &args[2],
1552             None => bcx.sess().span_bug(call_info.span,
1553                                         "intrinsic call with unexpected argument shape"),
1554         };
1555         let vector = consts::const_expr(bcx.ccx(), vector, tcx.mk_substs(substs), None).0;
1556
1557         let indices: Option<Vec<_>> = (0..n)
1558             .map(|i| {
1559                 let arg_idx = i;
1560                 let val = const_get_elt(bcx.ccx(), vector, &[i as libc::c_uint]);
1561                 let c = const_to_opt_uint(val);
1562                 match c {
1563                     None => {
1564                         emit_error!("shuffle index #{} is not a constant", arg_idx);
1565                         None
1566                     }
1567                     Some(idx) if idx >= total_len => {
1568                         emit_error!("shuffle index #{} is out of bounds (limit {})",
1569                                     arg_idx, total_len);
1570                         None
1571                     }
1572                     Some(idx) => Some(C_i32(bcx.ccx(), idx as i32)),
1573                 }
1574             })
1575             .collect();
1576         let indices = match indices {
1577             Some(i) => i,
1578             None => return C_null(llret_ty)
1579         };
1580
1581         return ShuffleVector(bcx, llargs[0], llargs[1], C_vector(&indices))
1582     }
1583
1584     if name == "simd_insert" {
1585         require!(in_elem == arg_tys[2],
1586                  "expected inserted type `{}` (element of input `{}`), found `{}`",
1587                  in_elem, in_ty, arg_tys[2]);
1588         return InsertElement(bcx, llargs[0], llargs[2], llargs[1])
1589     }
1590     if name == "simd_extract" {
1591         require!(ret_ty == in_elem,
1592                  "expected return type `{}` (element of input `{}`), found `{}`",
1593                  in_elem, in_ty, ret_ty);
1594         return ExtractElement(bcx, llargs[0], llargs[1])
1595     }
1596
1597     if name == "simd_cast" {
1598         require_simd!(ret_ty, "return");
1599         let out_len = ret_ty.simd_size(tcx);
1600         require!(in_len == out_len,
1601                  "expected return type with length {} (same as input type `{}`), \
1602                   found `{}` with length {}",
1603                  in_len, in_ty,
1604                  ret_ty, out_len);
1605         // casting cares about nominal type, not just structural type
1606         let out_elem = ret_ty.simd_type(tcx);
1607
1608         if in_elem == out_elem { return llargs[0]; }
1609
1610         enum Style { Float, Int(/* is signed? */ bool), Unsupported }
1611
1612         let (in_style, in_width) = match in_elem.sty {
1613             // vectors of pointer-sized integers should've been
1614             // disallowed before here, so this unwrap is safe.
1615             ty::TyInt(i) => (Style::Int(true), i.bit_width().unwrap()),
1616             ty::TyUint(u) => (Style::Int(false), u.bit_width().unwrap()),
1617             ty::TyFloat(f) => (Style::Float, f.bit_width()),
1618             _ => (Style::Unsupported, 0)
1619         };
1620         let (out_style, out_width) = match out_elem.sty {
1621             ty::TyInt(i) => (Style::Int(true), i.bit_width().unwrap()),
1622             ty::TyUint(u) => (Style::Int(false), u.bit_width().unwrap()),
1623             ty::TyFloat(f) => (Style::Float, f.bit_width()),
1624             _ => (Style::Unsupported, 0)
1625         };
1626
1627         match (in_style, out_style) {
1628             (Style::Int(in_is_signed), Style::Int(_)) => {
1629                 return match in_width.cmp(&out_width) {
1630                     Ordering::Greater => Trunc(bcx, llargs[0], llret_ty),
1631                     Ordering::Equal => llargs[0],
1632                     Ordering::Less => if in_is_signed {
1633                         SExt(bcx, llargs[0], llret_ty)
1634                     } else {
1635                         ZExt(bcx, llargs[0], llret_ty)
1636                     }
1637                 }
1638             }
1639             (Style::Int(in_is_signed), Style::Float) => {
1640                 return if in_is_signed {
1641                     SIToFP(bcx, llargs[0], llret_ty)
1642                 } else {
1643                     UIToFP(bcx, llargs[0], llret_ty)
1644                 }
1645             }
1646             (Style::Float, Style::Int(out_is_signed)) => {
1647                 return if out_is_signed {
1648                     FPToSI(bcx, llargs[0], llret_ty)
1649                 } else {
1650                     FPToUI(bcx, llargs[0], llret_ty)
1651                 }
1652             }
1653             (Style::Float, Style::Float) => {
1654                 return match in_width.cmp(&out_width) {
1655                     Ordering::Greater => FPTrunc(bcx, llargs[0], llret_ty),
1656                     Ordering::Equal => llargs[0],
1657                     Ordering::Less => FPExt(bcx, llargs[0], llret_ty)
1658                 }
1659             }
1660             _ => {/* Unsupported. Fallthrough. */}
1661         }
1662         require!(false,
1663                  "unsupported cast from `{}` with element `{}` to `{}` with element `{}`",
1664                  in_ty, in_elem,
1665                  ret_ty, out_elem);
1666     }
1667     macro_rules! arith {
1668         ($($name: ident: $($($p: ident),* => $call: expr),*;)*) => {
1669             $(
1670                 if name == stringify!($name) {
1671                     match in_elem.sty {
1672                         $(
1673                             $(ty::$p(_))|* => {
1674                                 return $call(bcx, llargs[0], llargs[1], call_debug_location)
1675                             }
1676                             )*
1677                         _ => {},
1678                     }
1679                     require!(false,
1680                              "unsupported operation on `{}` with element `{}`",
1681                              in_ty,
1682                              in_elem)
1683                 })*
1684         }
1685     }
1686     arith! {
1687         simd_add: TyUint, TyInt => Add, TyFloat => FAdd;
1688         simd_sub: TyUint, TyInt => Sub, TyFloat => FSub;
1689         simd_mul: TyUint, TyInt => Mul, TyFloat => FMul;
1690         simd_div: TyFloat => FDiv;
1691         simd_shl: TyUint, TyInt => Shl;
1692         simd_shr: TyUint => LShr, TyInt => AShr;
1693         simd_and: TyUint, TyInt => And;
1694         simd_or: TyUint, TyInt => Or;
1695         simd_xor: TyUint, TyInt => Xor;
1696     }
1697     bcx.sess().span_bug(call_info.span, "unknown SIMD intrinsic");
1698 }