compiler/rustc_codegen_llvm/src/builder.rs

   1 use crate::common::Funclet;
   2 use crate::context::CodegenCx;
   3 use crate::llvm::{self, BasicBlock, False};
   4 use crate::llvm::{AtomicOrdering, AtomicRmwBinOp, SynchronizationScope};
   5 use crate::llvm_util;
   6 use crate::type_::Type;
   7 use crate::type_of::LayoutLlvmExt;
   8 use crate::value::Value;
   9 use cstr::cstr;
  10 use libc::{c_char, c_uint};
  11 use rustc_codegen_ssa::common::{IntPredicate, RealPredicate, TypeKind};
  12 use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue};
  13 use rustc_codegen_ssa::mir::place::PlaceRef;
  14 use rustc_codegen_ssa::traits::*;
  15 use rustc_codegen_ssa::MemFlags;
  16 use rustc_data_structures::small_c_str::SmallCStr;
  17 use rustc_hir::def_id::DefId;
  18 use rustc_middle::ty::layout::TyAndLayout;
  19 use rustc_middle::ty::{self, Ty, TyCtxt};
  20 use rustc_span::Span;
  21 use rustc_target::abi::{self, Align, Size};
  22 use rustc_target::spec::{HasTargetSpec, Target};
  23 use std::borrow::Cow;
  24 use std::ffi::CStr;
  25 use std::iter;
  26 use std::ops::{Deref, Range};
  27 use std::ptr;
  28 use tracing::debug;
  29
  30 // All Builders must have an llfn associated with them
  31 #[must_use]
  32 pub struct Builder<'a, 'll, 'tcx> {
  33     pub llbuilder: &'ll mut llvm::Builder<'ll>,
  34     pub cx: &'a CodegenCx<'ll, 'tcx>,
  35 }
  36
  37 impl Drop for Builder<'a, 'll, 'tcx> {
  38     fn drop(&mut self) {
  39         unsafe {
  40             llvm::LLVMDisposeBuilder(&mut *(self.llbuilder as *mut _));
  41         }
  42     }
  43 }
  44
  45 // FIXME(eddyb) use a checked constructor when they become `const fn`.
  46 const EMPTY_C_STR: &CStr = unsafe { CStr::from_bytes_with_nul_unchecked(b"\0") };
  47
  48 /// Empty string, to be used where LLVM expects an instruction name, indicating
  49 /// that the instruction is to be left unnamed (i.e. numbered, in textual IR).
  50 // FIXME(eddyb) pass `&CStr` directly to FFI once it's a thin pointer.
  51 const UNNAMED: *const c_char = EMPTY_C_STR.as_ptr();
  52
  53 impl BackendTypes for Builder<'_, 'll, 'tcx> {
  54     type Value = <CodegenCx<'ll, 'tcx> as BackendTypes>::Value;
  55     type Function = <CodegenCx<'ll, 'tcx> as BackendTypes>::Function;
  56     type BasicBlock = <CodegenCx<'ll, 'tcx> as BackendTypes>::BasicBlock;
  57     type Type = <CodegenCx<'ll, 'tcx> as BackendTypes>::Type;
  58     type Funclet = <CodegenCx<'ll, 'tcx> as BackendTypes>::Funclet;
  59
  60     type DIScope = <CodegenCx<'ll, 'tcx> as BackendTypes>::DIScope;
  61     type DILocation = <CodegenCx<'ll, 'tcx> as BackendTypes>::DILocation;
  62     type DIVariable = <CodegenCx<'ll, 'tcx> as BackendTypes>::DIVariable;
  63 }
  64
  65 impl abi::HasDataLayout for Builder<'_, '_, '_> {
  66     fn data_layout(&self) -> &abi::TargetDataLayout {
  67         self.cx.data_layout()
  68     }
  69 }
  70
  71 impl ty::layout::HasTyCtxt<'tcx> for Builder<'_, '_, 'tcx> {
  72     #[inline]
  73     fn tcx(&self) -> TyCtxt<'tcx> {
  74         self.cx.tcx
  75     }
  76 }
  77
  78 impl ty::layout::HasParamEnv<'tcx> for Builder<'_, '_, 'tcx> {
  79     fn param_env(&self) -> ty::ParamEnv<'tcx> {
  80         self.cx.param_env()
  81     }
  82 }
  83
  84 impl HasTargetSpec for Builder<'_, '_, 'tcx> {
  85     #[inline]
  86     fn target_spec(&self) -> &Target {
  87         &self.cx.target_spec()
  88     }
  89 }
  90
  91 impl abi::LayoutOf for Builder<'_, '_, 'tcx> {
  92     type Ty = Ty<'tcx>;
  93     type TyAndLayout = TyAndLayout<'tcx>;
  94
  95     fn layout_of(&self, ty: Ty<'tcx>) -> Self::TyAndLayout {
  96         self.cx.layout_of(ty)
  97     }
  98 }
  99
 100 impl Deref for Builder<'_, 'll, 'tcx> {
 101     type Target = CodegenCx<'ll, 'tcx>;
 102
 103     #[inline]
 104     fn deref(&self) -> &Self::Target {
 105         self.cx
 106     }
 107 }
 108
 109 impl HasCodegen<'tcx> for Builder<'_, 'll, 'tcx> {
 110     type CodegenCx = CodegenCx<'ll, 'tcx>;
 111 }
 112
 113 macro_rules! builder_methods_for_value_instructions {
 114     ($($name:ident($($arg:ident),*) => $llvm_capi:ident),+ $(,)?) => {
 115         $(fn $name(&mut self, $($arg: &'ll Value),*) -> &'ll Value {
 116             unsafe {
 117                 llvm::$llvm_capi(self.llbuilder, $($arg,)* UNNAMED)
 118             }
 119         })+
 120     }
 121 }
 122
 123 impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
 124     fn build(cx: &'a CodegenCx<'ll, 'tcx>, llbb: &'ll BasicBlock) -> Self {
 125         let bx = Builder::with_cx(cx);
 126         unsafe {
 127             llvm::LLVMPositionBuilderAtEnd(bx.llbuilder, llbb);
 128         }
 129         bx
 130     }
 131
 132     fn cx(&self) -> &CodegenCx<'ll, 'tcx> {
 133         self.cx
 134     }
 135
 136     fn llbb(&self) -> &'ll BasicBlock {
 137         unsafe { llvm::LLVMGetInsertBlock(self.llbuilder) }
 138     }
 139
 140     fn set_span(&mut self, _span: Span) {}
 141
 142     fn append_block(cx: &'a CodegenCx<'ll, 'tcx>, llfn: &'ll Value, name: &str) -> &'ll BasicBlock {
 143         unsafe {
 144             let name = SmallCStr::new(name);
 145             llvm::LLVMAppendBasicBlockInContext(cx.llcx, llfn, name.as_ptr())
 146         }
 147     }
 148
 149     fn append_sibling_block(&mut self, name: &str) -> &'ll BasicBlock {
 150         Self::append_block(self.cx, self.llfn(), name)
 151     }
 152
 153     fn build_sibling_block(&mut self, name: &str) -> Self {
 154         let llbb = self.append_sibling_block(name);
 155         Self::build(self.cx, llbb)
 156     }
 157
 158     fn ret_void(&mut self) {
 159         unsafe {
 160             llvm::LLVMBuildRetVoid(self.llbuilder);
 161         }
 162     }
 163
 164     fn ret(&mut self, v: &'ll Value) {
 165         unsafe {
 166             llvm::LLVMBuildRet(self.llbuilder, v);
 167         }
 168     }
 169
 170     fn br(&mut self, dest: &'ll BasicBlock) {
 171         unsafe {
 172             llvm::LLVMBuildBr(self.llbuilder, dest);
 173         }
 174     }
 175
 176     fn cond_br(
 177         &mut self,
 178         cond: &'ll Value,
 179         then_llbb: &'ll BasicBlock,
 180         else_llbb: &'ll BasicBlock,
 181     ) {
 182         unsafe {
 183             llvm::LLVMBuildCondBr(self.llbuilder, cond, then_llbb, else_llbb);
 184         }
 185     }
 186
 187     fn switch(
 188         &mut self,
 189         v: &'ll Value,
 190         else_llbb: &'ll BasicBlock,
 191         cases: impl ExactSizeIterator<Item = (u128, &'ll BasicBlock)>,
 192     ) {
 193         let switch =
 194             unsafe { llvm::LLVMBuildSwitch(self.llbuilder, v, else_llbb, cases.len() as c_uint) };
 195         for (on_val, dest) in cases {
 196             let on_val = self.const_uint_big(self.val_ty(v), on_val);
 197             unsafe { llvm::LLVMAddCase(switch, on_val, dest) }
 198         }
 199     }
 200
 201     fn invoke(
 202         &mut self,
 203         llfn: &'ll Value,
 204         args: &[&'ll Value],
 205         then: &'ll BasicBlock,
 206         catch: &'ll BasicBlock,
 207         funclet: Option<&Funclet<'ll>>,
 208     ) -> &'ll Value {
 209         debug!("invoke {:?} with args ({:?})", llfn, args);
 210
 211         let args = self.check_call("invoke", llfn, args);
 212         let bundle = funclet.map(|funclet| funclet.bundle());
 213         let bundle = bundle.as_ref().map(|b| &*b.raw);
 214
 215         unsafe {
 216             llvm::LLVMRustBuildInvoke(
 217                 self.llbuilder,
 218                 llfn,
 219                 args.as_ptr(),
 220                 args.len() as c_uint,
 221                 then,
 222                 catch,
 223                 bundle,
 224                 UNNAMED,
 225             )
 226         }
 227     }
 228
 229     fn unreachable(&mut self) {
 230         unsafe {
 231             llvm::LLVMBuildUnreachable(self.llbuilder);
 232         }
 233     }
 234
 235     builder_methods_for_value_instructions! {
 236         add(a, b) => LLVMBuildAdd,
 237         fadd(a, b) => LLVMBuildFAdd,
 238         sub(a, b) => LLVMBuildSub,
 239         fsub(a, b) => LLVMBuildFSub,
 240         mul(a, b) => LLVMBuildMul,
 241         fmul(a, b) => LLVMBuildFMul,
 242         udiv(a, b) => LLVMBuildUDiv,
 243         exactudiv(a, b) => LLVMBuildExactUDiv,
 244         sdiv(a, b) => LLVMBuildSDiv,
 245         exactsdiv(a, b) => LLVMBuildExactSDiv,
 246         fdiv(a, b) => LLVMBuildFDiv,
 247         urem(a, b) => LLVMBuildURem,
 248         srem(a, b) => LLVMBuildSRem,
 249         frem(a, b) => LLVMBuildFRem,
 250         shl(a, b) => LLVMBuildShl,
 251         lshr(a, b) => LLVMBuildLShr,
 252         ashr(a, b) => LLVMBuildAShr,
 253         and(a, b) => LLVMBuildAnd,
 254         or(a, b) => LLVMBuildOr,
 255         xor(a, b) => LLVMBuildXor,
 256         neg(x) => LLVMBuildNeg,
 257         fneg(x) => LLVMBuildFNeg,
 258         not(x) => LLVMBuildNot,
 259         unchecked_sadd(x, y) => LLVMBuildNSWAdd,
 260         unchecked_uadd(x, y) => LLVMBuildNUWAdd,
 261         unchecked_ssub(x, y) => LLVMBuildNSWSub,
 262         unchecked_usub(x, y) => LLVMBuildNUWSub,
 263         unchecked_smul(x, y) => LLVMBuildNSWMul,
 264         unchecked_umul(x, y) => LLVMBuildNUWMul,
 265     }
 266
 267     fn fadd_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
 268         unsafe {
 269             let instr = llvm::LLVMBuildFAdd(self.llbuilder, lhs, rhs, UNNAMED);
 270             llvm::LLVMRustSetFastMath(instr);
 271             instr
 272         }
 273     }
 274
 275     fn fsub_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
 276         unsafe {
 277             let instr = llvm::LLVMBuildFSub(self.llbuilder, lhs, rhs, UNNAMED);
 278             llvm::LLVMRustSetFastMath(instr);
 279             instr
 280         }
 281     }
 282
 283     fn fmul_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
 284         unsafe {
 285             let instr = llvm::LLVMBuildFMul(self.llbuilder, lhs, rhs, UNNAMED);
 286             llvm::LLVMRustSetFastMath(instr);
 287             instr
 288         }
 289     }
 290
 291     fn fdiv_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
 292         unsafe {
 293             let instr = llvm::LLVMBuildFDiv(self.llbuilder, lhs, rhs, UNNAMED);
 294             llvm::LLVMRustSetFastMath(instr);
 295             instr
 296         }
 297     }
 298
 299     fn frem_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
 300         unsafe {
 301             let instr = llvm::LLVMBuildFRem(self.llbuilder, lhs, rhs, UNNAMED);
 302             llvm::LLVMRustSetFastMath(instr);
 303             instr
 304         }
 305     }
 306
 307     fn checked_binop(
 308         &mut self,
 309         oop: OverflowOp,
 310         ty: Ty<'_>,
 311         lhs: Self::Value,
 312         rhs: Self::Value,
 313     ) -> (Self::Value, Self::Value) {
 314         use rustc_middle::ty::{Int, Uint};
 315         use rustc_middle::ty::{IntTy::*, UintTy::*};
 316
 317         let new_kind = match ty.kind() {
 318             Int(t @ Isize) => Int(t.normalize(self.tcx.sess.target.pointer_width)),
 319             Uint(t @ Usize) => Uint(t.normalize(self.tcx.sess.target.pointer_width)),
 320             t @ (Uint(_) | Int(_)) => t.clone(),
 321             _ => panic!("tried to get overflow intrinsic for op applied to non-int type"),
 322         };
 323
 324         let name = match oop {
 325             OverflowOp::Add => match new_kind {
 326                 Int(I8) => "llvm.sadd.with.overflow.i8",
 327                 Int(I16) => "llvm.sadd.with.overflow.i16",
 328                 Int(I32) => "llvm.sadd.with.overflow.i32",
 329                 Int(I64) => "llvm.sadd.with.overflow.i64",
 330                 Int(I128) => "llvm.sadd.with.overflow.i128",
 331
 332                 Uint(U8) => "llvm.uadd.with.overflow.i8",
 333                 Uint(U16) => "llvm.uadd.with.overflow.i16",
 334                 Uint(U32) => "llvm.uadd.with.overflow.i32",
 335                 Uint(U64) => "llvm.uadd.with.overflow.i64",
 336                 Uint(U128) => "llvm.uadd.with.overflow.i128",
 337
 338                 _ => unreachable!(),
 339             },
 340             OverflowOp::Sub => match new_kind {
 341                 Int(I8) => "llvm.ssub.with.overflow.i8",
 342                 Int(I16) => "llvm.ssub.with.overflow.i16",
 343                 Int(I32) => "llvm.ssub.with.overflow.i32",
 344                 Int(I64) => "llvm.ssub.with.overflow.i64",
 345                 Int(I128) => "llvm.ssub.with.overflow.i128",
 346
 347                 Uint(U8) => "llvm.usub.with.overflow.i8",
 348                 Uint(U16) => "llvm.usub.with.overflow.i16",
 349                 Uint(U32) => "llvm.usub.with.overflow.i32",
 350                 Uint(U64) => "llvm.usub.with.overflow.i64",
 351                 Uint(U128) => "llvm.usub.with.overflow.i128",
 352
 353                 _ => unreachable!(),
 354             },
 355             OverflowOp::Mul => match new_kind {
 356                 Int(I8) => "llvm.smul.with.overflow.i8",
 357                 Int(I16) => "llvm.smul.with.overflow.i16",
 358                 Int(I32) => "llvm.smul.with.overflow.i32",
 359                 Int(I64) => "llvm.smul.with.overflow.i64",
 360                 Int(I128) => "llvm.smul.with.overflow.i128",
 361
 362                 Uint(U8) => "llvm.umul.with.overflow.i8",
 363                 Uint(U16) => "llvm.umul.with.overflow.i16",
 364                 Uint(U32) => "llvm.umul.with.overflow.i32",
 365                 Uint(U64) => "llvm.umul.with.overflow.i64",
 366                 Uint(U128) => "llvm.umul.with.overflow.i128",
 367
 368                 _ => unreachable!(),
 369             },
 370         };
 371
 372         let intrinsic = self.get_intrinsic(&name);
 373         let res = self.call(intrinsic, &[lhs, rhs], None);
 374         (self.extract_value(res, 0), self.extract_value(res, 1))
 375     }
 376
 377     fn from_immediate(&mut self, val: Self::Value) -> Self::Value {
 378         if self.cx().val_ty(val) == self.cx().type_i1() {
 379             self.zext(val, self.cx().type_i8())
 380         } else {
 381             val
 382         }
 383     }
 384     fn to_immediate_scalar(&mut self, val: Self::Value, scalar: &abi::Scalar) -> Self::Value {
 385         if scalar.is_bool() {
 386             return self.trunc(val, self.cx().type_i1());
 387         }
 388         val
 389     }
 390
 391     fn alloca(&mut self, ty: &'ll Type, align: Align) -> &'ll Value {
 392         let mut bx = Builder::with_cx(self.cx);
 393         bx.position_at_start(unsafe { llvm::LLVMGetFirstBasicBlock(self.llfn()) });
 394         bx.dynamic_alloca(ty, align)
 395     }
 396
 397     fn dynamic_alloca(&mut self, ty: &'ll Type, align: Align) -> &'ll Value {
 398         unsafe {
 399             let alloca = llvm::LLVMBuildAlloca(self.llbuilder, ty, UNNAMED);
 400             llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint);
 401             alloca
 402         }
 403     }
 404
 405     fn array_alloca(&mut self, ty: &'ll Type, len: &'ll Value, align: Align) -> &'ll Value {
 406         unsafe {
 407             let alloca = llvm::LLVMBuildArrayAlloca(self.llbuilder, ty, len, UNNAMED);
 408             llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint);
 409             alloca
 410         }
 411     }
 412
 413     fn load(&mut self, ty: &'ll Type, ptr: &'ll Value, align: Align) -> &'ll Value {
 414         unsafe {
 415             let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED);
 416             llvm::LLVMSetAlignment(load, align.bytes() as c_uint);
 417             load
 418         }
 419     }
 420
 421     fn volatile_load(&mut self, ty: &'ll Type, ptr: &'ll Value) -> &'ll Value {
 422         unsafe {
 423             let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED);
 424             llvm::LLVMSetVolatile(load, llvm::True);
 425             load
 426         }
 427     }
 428
 429     fn atomic_load(
 430         &mut self,
 431         ty: &'ll Type,
 432         ptr: &'ll Value,
 433         order: rustc_codegen_ssa::common::AtomicOrdering,
 434         size: Size,
 435     ) -> &'ll Value {
 436         unsafe {
 437             let load = llvm::LLVMRustBuildAtomicLoad(
 438                 self.llbuilder,
 439                 ty,
 440                 ptr,
 441                 UNNAMED,
 442                 AtomicOrdering::from_generic(order),
 443             );
 444             // LLVM requires the alignment of atomic loads to be at least the size of the type.
 445             llvm::LLVMSetAlignment(load, size.bytes() as c_uint);
 446             load
 447         }
 448     }
 449
 450     fn load_operand(&mut self, place: PlaceRef<'tcx, &'ll Value>) -> OperandRef<'tcx, &'ll Value> {
 451         debug!("PlaceRef::load: {:?}", place);
 452
 453         assert_eq!(place.llextra.is_some(), place.layout.is_unsized());
 454
 455         if place.layout.is_zst() {
 456             return OperandRef::new_zst(self, place.layout);
 457         }
 458
 459         fn scalar_load_metadata<'a, 'll, 'tcx>(
 460             bx: &mut Builder<'a, 'll, 'tcx>,
 461             load: &'ll Value,
 462             scalar: &abi::Scalar,
 463         ) {
 464             let vr = scalar.valid_range.clone();
 465             match scalar.value {
 466                 abi::Int(..) => {
 467                     let range = scalar.valid_range_exclusive(bx);
 468                     if range.start != range.end {
 469                         bx.range_metadata(load, range);
 470                     }
 471                 }
 472                 abi::Pointer if vr.start() < vr.end() && !vr.contains(&0) => {
 473                     bx.nonnull_metadata(load);
 474                 }
 475                 _ => {}
 476             }
 477         }
 478
 479         let val = if let Some(llextra) = place.llextra {
 480             OperandValue::Ref(place.llval, Some(llextra), place.align)
 481         } else if place.layout.is_llvm_immediate() {
 482             let mut const_llval = None;
 483             unsafe {
 484                 if let Some(global) = llvm::LLVMIsAGlobalVariable(place.llval) {
 485                     if llvm::LLVMIsGlobalConstant(global) == llvm::True {
 486                         const_llval = llvm::LLVMGetInitializer(global);
 487                     }
 488                 }
 489             }
 490             let llval = const_llval.unwrap_or_else(|| {
 491                 let load = self.load(place.layout.llvm_type(self), place.llval, place.align);
 492                 if let abi::Abi::Scalar(ref scalar) = place.layout.abi {
 493                     scalar_load_metadata(self, load, scalar);
 494                 }
 495                 load
 496             });
 497             OperandValue::Immediate(self.to_immediate(llval, place.layout))
 498         } else if let abi::Abi::ScalarPair(ref a, ref b) = place.layout.abi {
 499             let b_offset = a.value.size(self).align_to(b.value.align(self).abi);
 500
 501             let mut load = |i, scalar: &abi::Scalar, align| {
 502                 let llptr = self.struct_gep(place.llval, i as u64);
 503                 let llty = place.layout.scalar_pair_element_llvm_type(self, i, false);
 504                 let load = self.load(llty, llptr, align);
 505                 scalar_load_metadata(self, load, scalar);
 506                 self.to_immediate_scalar(load, scalar)
 507             };
 508
 509             OperandValue::Pair(
 510                 load(0, a, place.align),
 511                 load(1, b, place.align.restrict_for_offset(b_offset)),
 512             )
 513         } else {
 514             OperandValue::Ref(place.llval, None, place.align)
 515         };
 516
 517         OperandRef { val, layout: place.layout }
 518     }
 519
 520     fn write_operand_repeatedly(
 521         mut self,
 522         cg_elem: OperandRef<'tcx, &'ll Value>,
 523         count: u64,
 524         dest: PlaceRef<'tcx, &'ll Value>,
 525     ) -> Self {
 526         let zero = self.const_usize(0);
 527         let count = self.const_usize(count);
 528         let start = dest.project_index(&mut self, zero).llval;
 529         let end = dest.project_index(&mut self, count).llval;
 530
 531         let mut header_bx = self.build_sibling_block("repeat_loop_header");
 532         let mut body_bx = self.build_sibling_block("repeat_loop_body");
 533         let next_bx = self.build_sibling_block("repeat_loop_next");
 534
 535         self.br(header_bx.llbb());
 536         let current = header_bx.phi(self.val_ty(start), &[start], &[self.llbb()]);
 537
 538         let keep_going = header_bx.icmp(IntPredicate::IntNE, current, end);
 539         header_bx.cond_br(keep_going, body_bx.llbb(), next_bx.llbb());
 540
 541         let align = dest.align.restrict_for_offset(dest.layout.field(self.cx(), 0).size);
 542         cg_elem
 543             .val
 544             .store(&mut body_bx, PlaceRef::new_sized_aligned(current, cg_elem.layout, align));
 545
 546         let next = body_bx.inbounds_gep(current, &[self.const_usize(1)]);
 547         body_bx.br(header_bx.llbb());
 548         header_bx.add_incoming_to_phi(current, next, body_bx.llbb());
 549
 550         next_bx
 551     }
 552
 553     fn range_metadata(&mut self, load: &'ll Value, range: Range<u128>) {
 554         if self.sess().target.arch == "amdgpu" {
 555             // amdgpu/LLVM does something weird and thinks a i64 value is
 556             // split into a v2i32, halving the bitwidth LLVM expects,
 557             // tripping an assertion. So, for now, just disable this
 558             // optimization.
 559             return;
 560         }
 561
 562         unsafe {
 563             let llty = self.cx.val_ty(load);
 564             let v = [
 565                 self.cx.const_uint_big(llty, range.start),
 566                 self.cx.const_uint_big(llty, range.end),
 567             ];
 568
 569             llvm::LLVMSetMetadata(
 570                 load,
 571                 llvm::MD_range as c_uint,
 572                 llvm::LLVMMDNodeInContext(self.cx.llcx, v.as_ptr(), v.len() as c_uint),
 573             );
 574         }
 575     }
 576
 577     fn nonnull_metadata(&mut self, load: &'ll Value) {
 578         unsafe {
 579             llvm::LLVMSetMetadata(
 580                 load,
 581                 llvm::MD_nonnull as c_uint,
 582                 llvm::LLVMMDNodeInContext(self.cx.llcx, ptr::null(), 0),
 583             );
 584         }
 585     }
 586
 587     fn store(&mut self, val: &'ll Value, ptr: &'ll Value, align: Align) -> &'ll Value {
 588         self.store_with_flags(val, ptr, align, MemFlags::empty())
 589     }
 590
 591     fn store_with_flags(
 592         &mut self,
 593         val: &'ll Value,
 594         ptr: &'ll Value,
 595         align: Align,
 596         flags: MemFlags,
 597     ) -> &'ll Value {
 598         debug!("Store {:?} -> {:?} ({:?})", val, ptr, flags);
 599         let ptr = self.check_store(val, ptr);
 600         unsafe {
 601             let store = llvm::LLVMBuildStore(self.llbuilder, val, ptr);
 602             let align =
 603                 if flags.contains(MemFlags::UNALIGNED) { 1 } else { align.bytes() as c_uint };
 604             llvm::LLVMSetAlignment(store, align);
 605             if flags.contains(MemFlags::VOLATILE) {
 606                 llvm::LLVMSetVolatile(store, llvm::True);
 607             }
 608             if flags.contains(MemFlags::NONTEMPORAL) {
 609                 // According to LLVM [1] building a nontemporal store must
 610                 // *always* point to a metadata value of the integer 1.
 611                 //
 612                 // [1]: https://llvm.org/docs/LangRef.html#store-instruction
 613                 let one = self.cx.const_i32(1);
 614                 let node = llvm::LLVMMDNodeInContext(self.cx.llcx, &one, 1);
 615                 llvm::LLVMSetMetadata(store, llvm::MD_nontemporal as c_uint, node);
 616             }
 617             store
 618         }
 619     }
 620
 621     fn atomic_store(
 622         &mut self,
 623         val: &'ll Value,
 624         ptr: &'ll Value,
 625         order: rustc_codegen_ssa::common::AtomicOrdering,
 626         size: Size,
 627     ) {
 628         debug!("Store {:?} -> {:?}", val, ptr);
 629         let ptr = self.check_store(val, ptr);
 630         unsafe {
 631             let store = llvm::LLVMRustBuildAtomicStore(
 632                 self.llbuilder,
 633                 val,
 634                 ptr,
 635                 AtomicOrdering::from_generic(order),
 636             );
 637             // LLVM requires the alignment of atomic stores to be at least the size of the type.
 638             llvm::LLVMSetAlignment(store, size.bytes() as c_uint);
 639         }
 640     }
 641
 642     fn gep(&mut self, ptr: &'ll Value, indices: &[&'ll Value]) -> &'ll Value {
 643         unsafe {
 644             llvm::LLVMBuildGEP(
 645                 self.llbuilder,
 646                 ptr,
 647                 indices.as_ptr(),
 648                 indices.len() as c_uint,
 649                 UNNAMED,
 650             )
 651         }
 652     }
 653
 654     fn inbounds_gep(&mut self, ptr: &'ll Value, indices: &[&'ll Value]) -> &'ll Value {
 655         unsafe {
 656             llvm::LLVMBuildInBoundsGEP(
 657                 self.llbuilder,
 658                 ptr,
 659                 indices.as_ptr(),
 660                 indices.len() as c_uint,
 661                 UNNAMED,
 662             )
 663         }
 664     }
 665
 666     fn struct_gep(&mut self, ptr: &'ll Value, idx: u64) -> &'ll Value {
 667         assert_eq!(idx as c_uint as u64, idx);
 668         unsafe { llvm::LLVMBuildStructGEP(self.llbuilder, ptr, idx as c_uint, UNNAMED) }
 669     }
 670
 671     /* Casts */
 672     fn trunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
 673         unsafe { llvm::LLVMBuildTrunc(self.llbuilder, val, dest_ty, UNNAMED) }
 674     }
 675
 676     fn sext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
 677         unsafe { llvm::LLVMBuildSExt(self.llbuilder, val, dest_ty, UNNAMED) }
 678     }
 679
 680     fn fptoui_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> Option<&'ll Value> {
 681         if llvm_util::get_version() >= (12, 0, 0) && !self.fptoint_sat_broken_in_llvm() {
 682             let src_ty = self.cx.val_ty(val);
 683             let float_width = self.cx.float_width(src_ty);
 684             let int_width = self.cx.int_width(dest_ty);
 685             let name = format!("llvm.fptoui.sat.i{}.f{}", int_width, float_width);
 686             let intrinsic = self.get_intrinsic(&name);
 687             return Some(self.call(intrinsic, &[val], None));
 688         }
 689
 690         None
 691     }
 692
 693     fn fptosi_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> Option<&'ll Value> {
 694         if llvm_util::get_version() >= (12, 0, 0) && !self.fptoint_sat_broken_in_llvm() {
 695             let src_ty = self.cx.val_ty(val);
 696             let float_width = self.cx.float_width(src_ty);
 697             let int_width = self.cx.int_width(dest_ty);
 698             let name = format!("llvm.fptosi.sat.i{}.f{}", int_width, float_width);
 699             let intrinsic = self.get_intrinsic(&name);
 700             return Some(self.call(intrinsic, &[val], None));
 701         }
 702
 703         None
 704     }
 705
 706     fn fptoui(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
 707         // On WebAssembly the `fptoui` and `fptosi` instructions currently have
 708         // poor codegen. The reason for this is that the corresponding wasm
 709         // instructions, `i32.trunc_f32_s` for example, will trap when the float
 710         // is out-of-bounds, infinity, or nan. This means that LLVM
 711         // automatically inserts control flow around `fptoui` and `fptosi`
 712         // because the LLVM instruction `fptoui` is defined as producing a
 713         // poison value, not having UB on out-of-bounds values.
 714         //
 715         // This method, however, is only used with non-saturating casts that
 716         // have UB on out-of-bounds values. This means that it's ok if we use
 717         // the raw wasm instruction since out-of-bounds values can do whatever
 718         // we like. To ensure that LLVM picks the right instruction we choose
 719         // the raw wasm intrinsic functions which avoid LLVM inserting all the
 720         // other control flow automatically.
 721         if self.sess().target.arch == "wasm32" {
 722             let src_ty = self.cx.val_ty(val);
 723             if self.cx.type_kind(src_ty) != TypeKind::Vector {
 724                 let float_width = self.cx.float_width(src_ty);
 725                 let int_width = self.cx.int_width(dest_ty);
 726                 let name = match (int_width, float_width) {
 727                     (32, 32) => Some("llvm.wasm.trunc.unsigned.i32.f32"),
 728                     (32, 64) => Some("llvm.wasm.trunc.unsigned.i32.f64"),
 729                     (64, 32) => Some("llvm.wasm.trunc.unsigned.i64.f32"),
 730                     (64, 64) => Some("llvm.wasm.trunc.unsigned.i64.f64"),
 731                     _ => None,
 732                 };
 733                 if let Some(name) = name {
 734                     let intrinsic = self.get_intrinsic(name);
 735                     return self.call(intrinsic, &[val], None);
 736                 }
 737             }
 738         }
 739         unsafe { llvm::LLVMBuildFPToUI(self.llbuilder, val, dest_ty, UNNAMED) }
 740     }
 741
 742     fn fptosi(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
 743         // see `fptoui` above for why wasm is different here
 744         if self.sess().target.arch == "wasm32" {
 745             let src_ty = self.cx.val_ty(val);
 746             if self.cx.type_kind(src_ty) != TypeKind::Vector {
 747                 let float_width = self.cx.float_width(src_ty);
 748                 let int_width = self.cx.int_width(dest_ty);
 749                 let name = match (int_width, float_width) {
 750                     (32, 32) => Some("llvm.wasm.trunc.signed.i32.f32"),
 751                     (32, 64) => Some("llvm.wasm.trunc.signed.i32.f64"),
 752                     (64, 32) => Some("llvm.wasm.trunc.signed.i64.f32"),
 753                     (64, 64) => Some("llvm.wasm.trunc.signed.i64.f64"),
 754                     _ => None,
 755                 };
 756                 if let Some(name) = name {
 757                     let intrinsic = self.get_intrinsic(name);
 758                     return self.call(intrinsic, &[val], None);
 759                 }
 760             }
 761         }
 762         unsafe { llvm::LLVMBuildFPToSI(self.llbuilder, val, dest_ty, UNNAMED) }
 763     }
 764
 765     fn uitofp(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
 766         unsafe { llvm::LLVMBuildUIToFP(self.llbuilder, val, dest_ty, UNNAMED) }
 767     }
 768
 769     fn sitofp(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
 770         unsafe { llvm::LLVMBuildSIToFP(self.llbuilder, val, dest_ty, UNNAMED) }
 771     }
 772
 773     fn fptrunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
 774         unsafe { llvm::LLVMBuildFPTrunc(self.llbuilder, val, dest_ty, UNNAMED) }
 775     }
 776
 777     fn fpext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
 778         unsafe { llvm::LLVMBuildFPExt(self.llbuilder, val, dest_ty, UNNAMED) }
 779     }
 780
 781     fn ptrtoint(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
 782         unsafe { llvm::LLVMBuildPtrToInt(self.llbuilder, val, dest_ty, UNNAMED) }
 783     }
 784
 785     fn inttoptr(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
 786         unsafe { llvm::LLVMBuildIntToPtr(self.llbuilder, val, dest_ty, UNNAMED) }
 787     }
 788
 789     fn bitcast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
 790         unsafe { llvm::LLVMBuildBitCast(self.llbuilder, val, dest_ty, UNNAMED) }
 791     }
 792
 793     fn intcast(&mut self, val: &'ll Value, dest_ty: &'ll Type, is_signed: bool) -> &'ll Value {
 794         unsafe { llvm::LLVMRustBuildIntCast(self.llbuilder, val, dest_ty, is_signed) }
 795     }
 796
 797     fn pointercast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
 798         unsafe { llvm::LLVMBuildPointerCast(self.llbuilder, val, dest_ty, UNNAMED) }
 799     }
 800
 801     /* Comparisons */
 802     fn icmp(&mut self, op: IntPredicate, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
 803         let op = llvm::IntPredicate::from_generic(op);
 804         unsafe { llvm::LLVMBuildICmp(self.llbuilder, op as c_uint, lhs, rhs, UNNAMED) }
 805     }
 806
 807     fn fcmp(&mut self, op: RealPredicate, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
 808         unsafe { llvm::LLVMBuildFCmp(self.llbuilder, op as c_uint, lhs, rhs, UNNAMED) }
 809     }
 810
 811     /* Miscellaneous instructions */
 812     fn memcpy(
 813         &mut self,
 814         dst: &'ll Value,
 815         dst_align: Align,
 816         src: &'ll Value,
 817         src_align: Align,
 818         size: &'ll Value,
 819         flags: MemFlags,
 820     ) {
 821         assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memcpy not supported");
 822         let size = self.intcast(size, self.type_isize(), false);
 823         let is_volatile = flags.contains(MemFlags::VOLATILE);
 824         let dst = self.pointercast(dst, self.type_i8p());
 825         let src = self.pointercast(src, self.type_i8p());
 826         unsafe {
 827             llvm::LLVMRustBuildMemCpy(
 828                 self.llbuilder,
 829                 dst,
 830                 dst_align.bytes() as c_uint,
 831                 src,
 832                 src_align.bytes() as c_uint,
 833                 size,
 834                 is_volatile,
 835             );
 836         }
 837     }
 838
 839     fn memmove(
 840         &mut self,
 841         dst: &'ll Value,
 842         dst_align: Align,
 843         src: &'ll Value,
 844         src_align: Align,
 845         size: &'ll Value,
 846         flags: MemFlags,
 847     ) {
 848         assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memmove not supported");
 849         let size = self.intcast(size, self.type_isize(), false);
 850         let is_volatile = flags.contains(MemFlags::VOLATILE);
 851         let dst = self.pointercast(dst, self.type_i8p());
 852         let src = self.pointercast(src, self.type_i8p());
 853         unsafe {
 854             llvm::LLVMRustBuildMemMove(
 855                 self.llbuilder,
 856                 dst,
 857                 dst_align.bytes() as c_uint,
 858                 src,
 859                 src_align.bytes() as c_uint,
 860                 size,
 861                 is_volatile,
 862             );
 863         }
 864     }
 865
 866     fn memset(
 867         &mut self,
 868         ptr: &'ll Value,
 869         fill_byte: &'ll Value,
 870         size: &'ll Value,
 871         align: Align,
 872         flags: MemFlags,
 873     ) {
 874         let is_volatile = flags.contains(MemFlags::VOLATILE);
 875         let ptr = self.pointercast(ptr, self.type_i8p());
 876         unsafe {
 877             llvm::LLVMRustBuildMemSet(
 878                 self.llbuilder,
 879                 ptr,
 880                 align.bytes() as c_uint,
 881                 fill_byte,
 882                 size,
 883                 is_volatile,
 884             );
 885         }
 886     }
 887
 888     fn select(
 889         &mut self,
 890         cond: &'ll Value,
 891         then_val: &'ll Value,
 892         else_val: &'ll Value,
 893     ) -> &'ll Value {
 894         unsafe { llvm::LLVMBuildSelect(self.llbuilder, cond, then_val, else_val, UNNAMED) }
 895     }
 896
 897     fn va_arg(&mut self, list: &'ll Value, ty: &'ll Type) -> &'ll Value {
 898         unsafe { llvm::LLVMBuildVAArg(self.llbuilder, list, ty, UNNAMED) }
 899     }
 900
 901     fn extract_element(&mut self, vec: &'ll Value, idx: &'ll Value) -> &'ll Value {
 902         unsafe { llvm::LLVMBuildExtractElement(self.llbuilder, vec, idx, UNNAMED) }
 903     }
 904
 905     fn vector_splat(&mut self, num_elts: usize, elt: &'ll Value) -> &'ll Value {
 906         unsafe {
 907             let elt_ty = self.cx.val_ty(elt);
 908             let undef = llvm::LLVMGetUndef(self.type_vector(elt_ty, num_elts as u64));
 909             let vec = self.insert_element(undef, elt, self.cx.const_i32(0));
 910             let vec_i32_ty = self.type_vector(self.type_i32(), num_elts as u64);
 911             self.shuffle_vector(vec, undef, self.const_null(vec_i32_ty))
 912         }
 913     }
 914
 915     fn extract_value(&mut self, agg_val: &'ll Value, idx: u64) -> &'ll Value {
 916         assert_eq!(idx as c_uint as u64, idx);
 917         unsafe { llvm::LLVMBuildExtractValue(self.llbuilder, agg_val, idx as c_uint, UNNAMED) }
 918     }
 919
 920     fn insert_value(&mut self, agg_val: &'ll Value, elt: &'ll Value, idx: u64) -> &'ll Value {
 921         assert_eq!(idx as c_uint as u64, idx);
 922         unsafe { llvm::LLVMBuildInsertValue(self.llbuilder, agg_val, elt, idx as c_uint, UNNAMED) }
 923     }
 924
 925     fn landing_pad(
 926         &mut self,
 927         ty: &'ll Type,
 928         pers_fn: &'ll Value,
 929         num_clauses: usize,
 930     ) -> &'ll Value {
 931         // Use LLVMSetPersonalityFn to set the personality. It supports arbitrary Consts while,
 932         // LLVMBuildLandingPad requires the argument to be a Function (as of LLVM 12). The
 933         // personality lives on the parent function anyway.
 934         self.set_personality_fn(pers_fn);
 935         unsafe {
 936             llvm::LLVMBuildLandingPad(self.llbuilder, ty, None, num_clauses as c_uint, UNNAMED)
 937         }
 938     }
 939
 940     fn set_cleanup(&mut self, landing_pad: &'ll Value) {
 941         unsafe {
 942             llvm::LLVMSetCleanup(landing_pad, llvm::True);
 943         }
 944     }
 945
 946     fn resume(&mut self, exn: &'ll Value) -> &'ll Value {
 947         unsafe { llvm::LLVMBuildResume(self.llbuilder, exn) }
 948     }
 949
 950     fn cleanup_pad(&mut self, parent: Option<&'ll Value>, args: &[&'ll Value]) -> Funclet<'ll> {
 951         let name = cstr!("cleanuppad");
 952         let ret = unsafe {
 953             llvm::LLVMRustBuildCleanupPad(
 954                 self.llbuilder,
 955                 parent,
 956                 args.len() as c_uint,
 957                 args.as_ptr(),
 958                 name.as_ptr(),
 959             )
 960         };
 961         Funclet::new(ret.expect("LLVM does not have support for cleanuppad"))
 962     }
 963
 964     fn cleanup_ret(
 965         &mut self,
 966         funclet: &Funclet<'ll>,
 967         unwind: Option<&'ll BasicBlock>,
 968     ) -> &'ll Value {
 969         let ret =
 970             unsafe { llvm::LLVMRustBuildCleanupRet(self.llbuilder, funclet.cleanuppad(), unwind) };
 971         ret.expect("LLVM does not have support for cleanupret")
 972     }
 973
 974     fn catch_pad(&mut self, parent: &'ll Value, args: &[&'ll Value]) -> Funclet<'ll> {
 975         let name = cstr!("catchpad");
 976         let ret = unsafe {
 977             llvm::LLVMRustBuildCatchPad(
 978                 self.llbuilder,
 979                 parent,
 980                 args.len() as c_uint,
 981                 args.as_ptr(),
 982                 name.as_ptr(),
 983             )
 984         };
 985         Funclet::new(ret.expect("LLVM does not have support for catchpad"))
 986     }
 987
 988     fn catch_switch(
 989         &mut self,
 990         parent: Option<&'ll Value>,
 991         unwind: Option<&'ll BasicBlock>,
 992         num_handlers: usize,
 993     ) -> &'ll Value {
 994         let name = cstr!("catchswitch");
 995         let ret = unsafe {
 996             llvm::LLVMRustBuildCatchSwitch(
 997                 self.llbuilder,
 998                 parent,
 999                 unwind,
1000                 num_handlers as c_uint,
1001                 name.as_ptr(),
1002             )
1003         };
1004         ret.expect("LLVM does not have support for catchswitch")
1005     }
1006
1007     fn add_handler(&mut self, catch_switch: &'ll Value, handler: &'ll BasicBlock) {
1008         unsafe {
1009             llvm::LLVMRustAddHandler(catch_switch, handler);
1010         }
1011     }
1012
1013     fn set_personality_fn(&mut self, personality: &'ll Value) {
1014         unsafe {
1015             llvm::LLVMSetPersonalityFn(self.llfn(), personality);
1016         }
1017     }
1018
1019     // Atomic Operations
1020     fn atomic_cmpxchg(
1021         &mut self,
1022         dst: &'ll Value,
1023         cmp: &'ll Value,
1024         src: &'ll Value,
1025         order: rustc_codegen_ssa::common::AtomicOrdering,
1026         failure_order: rustc_codegen_ssa::common::AtomicOrdering,
1027         weak: bool,
1028     ) -> &'ll Value {
1029         let weak = if weak { llvm::True } else { llvm::False };
1030         unsafe {
1031             llvm::LLVMRustBuildAtomicCmpXchg(
1032                 self.llbuilder,
1033                 dst,
1034                 cmp,
1035                 src,
1036                 AtomicOrdering::from_generic(order),
1037                 AtomicOrdering::from_generic(failure_order),
1038                 weak,
1039             )
1040         }
1041     }
1042     fn atomic_rmw(
1043         &mut self,
1044         op: rustc_codegen_ssa::common::AtomicRmwBinOp,
1045         dst: &'ll Value,
1046         src: &'ll Value,
1047         order: rustc_codegen_ssa::common::AtomicOrdering,
1048     ) -> &'ll Value {
1049         unsafe {
1050             llvm::LLVMBuildAtomicRMW(
1051                 self.llbuilder,
1052                 AtomicRmwBinOp::from_generic(op),
1053                 dst,
1054                 src,
1055                 AtomicOrdering::from_generic(order),
1056                 False,
1057             )
1058         }
1059     }
1060
1061     fn atomic_fence(
1062         &mut self,
1063         order: rustc_codegen_ssa::common::AtomicOrdering,
1064         scope: rustc_codegen_ssa::common::SynchronizationScope,
1065     ) {
1066         unsafe {
1067             llvm::LLVMRustBuildAtomicFence(
1068                 self.llbuilder,
1069                 AtomicOrdering::from_generic(order),
1070                 SynchronizationScope::from_generic(scope),
1071             );
1072         }
1073     }
1074
1075     fn set_invariant_load(&mut self, load: &'ll Value) {
1076         unsafe {
1077             llvm::LLVMSetMetadata(
1078                 load,
1079                 llvm::MD_invariant_load as c_uint,
1080                 llvm::LLVMMDNodeInContext(self.cx.llcx, ptr::null(), 0),
1081             );
1082         }
1083     }
1084
1085     fn lifetime_start(&mut self, ptr: &'ll Value, size: Size) {
1086         self.call_lifetime_intrinsic("llvm.lifetime.start.p0i8", ptr, size);
1087     }
1088
1089     fn lifetime_end(&mut self, ptr: &'ll Value, size: Size) {
1090         self.call_lifetime_intrinsic("llvm.lifetime.end.p0i8", ptr, size);
1091     }
1092
1093     fn instrprof_increment(
1094         &mut self,
1095         fn_name: &'ll Value,
1096         hash: &'ll Value,
1097         num_counters: &'ll Value,
1098         index: &'ll Value,
1099     ) {
1100         debug!(
1101             "instrprof_increment() with args ({:?}, {:?}, {:?}, {:?})",
1102             fn_name, hash, num_counters, index
1103         );
1104
1105         let llfn = unsafe { llvm::LLVMRustGetInstrProfIncrementIntrinsic(self.cx().llmod) };
1106         let args = &[fn_name, hash, num_counters, index];
1107         let args = self.check_call("call", llfn, args);
1108
1109         unsafe {
1110             let _ = llvm::LLVMRustBuildCall(
1111                 self.llbuilder,
1112                 llfn,
1113                 args.as_ptr() as *const &llvm::Value,
1114                 args.len() as c_uint,
1115                 None,
1116             );
1117         }
1118     }
1119
1120     fn call(
1121         &mut self,
1122         llfn: &'ll Value,
1123         args: &[&'ll Value],
1124         funclet: Option<&Funclet<'ll>>,
1125     ) -> &'ll Value {
1126         debug!("call {:?} with args ({:?})", llfn, args);
1127
1128         let args = self.check_call("call", llfn, args);
1129         let bundle = funclet.map(|funclet| funclet.bundle());
1130         let bundle = bundle.as_ref().map(|b| &*b.raw);
1131
1132         unsafe {
1133             llvm::LLVMRustBuildCall(
1134                 self.llbuilder,
1135                 llfn,
1136                 args.as_ptr() as *const &llvm::Value,
1137                 args.len() as c_uint,
1138                 bundle,
1139             )
1140         }
1141     }
1142
1143     fn zext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
1144         unsafe { llvm::LLVMBuildZExt(self.llbuilder, val, dest_ty, UNNAMED) }
1145     }
1146
1147     fn do_not_inline(&mut self, llret: &'ll Value) {
1148         llvm::Attribute::NoInline.apply_callsite(llvm::AttributePlace::Function, llret);
1149     }
1150 }
1151
1152 impl StaticBuilderMethods for Builder<'a, 'll, 'tcx> {
1153     fn get_static(&mut self, def_id: DefId) -> &'ll Value {
1154         // Forward to the `get_static` method of `CodegenCx`
1155         self.cx().get_static(def_id)
1156     }
1157 }
1158
1159 impl Builder<'a, 'll, 'tcx> {
1160     fn with_cx(cx: &'a CodegenCx<'ll, 'tcx>) -> Self {
1161         // Create a fresh builder from the crate context.
1162         let llbuilder = unsafe { llvm::LLVMCreateBuilderInContext(cx.llcx) };
1163         Builder { llbuilder, cx }
1164     }
1165
1166     pub fn llfn(&self) -> &'ll Value {
1167         unsafe { llvm::LLVMGetBasicBlockParent(self.llbb()) }
1168     }
1169
1170     fn position_at_start(&mut self, llbb: &'ll BasicBlock) {
1171         unsafe {
1172             llvm::LLVMRustPositionBuilderAtStart(self.llbuilder, llbb);
1173         }
1174     }
1175
1176     pub fn minnum(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1177         unsafe { llvm::LLVMRustBuildMinNum(self.llbuilder, lhs, rhs) }
1178     }
1179
1180     pub fn maxnum(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
1181         unsafe { llvm::LLVMRustBuildMaxNum(self.llbuilder, lhs, rhs) }
1182     }
1183
1184     pub fn insert_element(
1185         &mut self,
1186         vec: &'ll Value,
1187         elt: &'ll Value,
1188         idx: &'ll Value,
1189     ) -> &'ll Value {
1190         unsafe { llvm::LLVMBuildInsertElement(self.llbuilder, vec, elt, idx, UNNAMED) }
1191     }
1192
1193     pub fn shuffle_vector(
1194         &mut self,
1195         v1: &'ll Value,
1196         v2: &'ll Value,
1197         mask: &'ll Value,
1198     ) -> &'ll Value {
1199         unsafe { llvm::LLVMBuildShuffleVector(self.llbuilder, v1, v2, mask, UNNAMED) }
1200     }
1201
1202     pub fn vector_reduce_fadd(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
1203         unsafe { llvm::LLVMRustBuildVectorReduceFAdd(self.llbuilder, acc, src) }
1204     }
1205     pub fn vector_reduce_fmul(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
1206         unsafe { llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src) }
1207     }
1208     pub fn vector_reduce_fadd_fast(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
1209         unsafe {
1210             let instr = llvm::LLVMRustBuildVectorReduceFAdd(self.llbuilder, acc, src);
1211             llvm::LLVMRustSetFastMath(instr);
1212             instr
1213         }
1214     }
1215     pub fn vector_reduce_fmul_fast(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value {
1216         unsafe {
1217             let instr = llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src);
1218             llvm::LLVMRustSetFastMath(instr);
1219             instr
1220         }
1221     }
1222     pub fn vector_reduce_add(&mut self, src: &'ll Value) -> &'ll Value {
1223         unsafe { llvm::LLVMRustBuildVectorReduceAdd(self.llbuilder, src) }
1224     }
1225     pub fn vector_reduce_mul(&mut self, src: &'ll Value) -> &'ll Value {
1226         unsafe { llvm::LLVMRustBuildVectorReduceMul(self.llbuilder, src) }
1227     }
1228     pub fn vector_reduce_and(&mut self, src: &'ll Value) -> &'ll Value {
1229         unsafe { llvm::LLVMRustBuildVectorReduceAnd(self.llbuilder, src) }
1230     }
1231     pub fn vector_reduce_or(&mut self, src: &'ll Value) -> &'ll Value {
1232         unsafe { llvm::LLVMRustBuildVectorReduceOr(self.llbuilder, src) }
1233     }
1234     pub fn vector_reduce_xor(&mut self, src: &'ll Value) -> &'ll Value {
1235         unsafe { llvm::LLVMRustBuildVectorReduceXor(self.llbuilder, src) }
1236     }
1237     pub fn vector_reduce_fmin(&mut self, src: &'ll Value) -> &'ll Value {
1238         unsafe {
1239             llvm::LLVMRustBuildVectorReduceFMin(self.llbuilder, src, /*NoNaNs:*/ false)
1240         }
1241     }
1242     pub fn vector_reduce_fmax(&mut self, src: &'ll Value) -> &'ll Value {
1243         unsafe {
1244             llvm::LLVMRustBuildVectorReduceFMax(self.llbuilder, src, /*NoNaNs:*/ false)
1245         }
1246     }
1247     pub fn vector_reduce_fmin_fast(&mut self, src: &'ll Value) -> &'ll Value {
1248         unsafe {
1249             let instr =
1250                 llvm::LLVMRustBuildVectorReduceFMin(self.llbuilder, src, /*NoNaNs:*/ true);
1251             llvm::LLVMRustSetFastMath(instr);
1252             instr
1253         }
1254     }
1255     pub fn vector_reduce_fmax_fast(&mut self, src: &'ll Value) -> &'ll Value {
1256         unsafe {
1257             let instr =
1258                 llvm::LLVMRustBuildVectorReduceFMax(self.llbuilder, src, /*NoNaNs:*/ true);
1259             llvm::LLVMRustSetFastMath(instr);
1260             instr
1261         }
1262     }
1263     pub fn vector_reduce_min(&mut self, src: &'ll Value, is_signed: bool) -> &'ll Value {
1264         unsafe { llvm::LLVMRustBuildVectorReduceMin(self.llbuilder, src, is_signed) }
1265     }
1266     pub fn vector_reduce_max(&mut self, src: &'ll Value, is_signed: bool) -> &'ll Value {
1267         unsafe { llvm::LLVMRustBuildVectorReduceMax(self.llbuilder, src, is_signed) }
1268     }
1269
1270     pub fn add_clause(&mut self, landing_pad: &'ll Value, clause: &'ll Value) {
1271         unsafe {
1272             llvm::LLVMAddClause(landing_pad, clause);
1273         }
1274     }
1275
1276     pub fn catch_ret(&mut self, funclet: &Funclet<'ll>, unwind: &'ll BasicBlock) -> &'ll Value {
1277         let ret =
1278             unsafe { llvm::LLVMRustBuildCatchRet(self.llbuilder, funclet.cleanuppad(), unwind) };
1279         ret.expect("LLVM does not have support for catchret")
1280     }
1281
1282     fn check_store(&mut self, val: &'ll Value, ptr: &'ll Value) -> &'ll Value {
1283         let dest_ptr_ty = self.cx.val_ty(ptr);
1284         let stored_ty = self.cx.val_ty(val);
1285         let stored_ptr_ty = self.cx.type_ptr_to(stored_ty);
1286
1287         assert_eq!(self.cx.type_kind(dest_ptr_ty), TypeKind::Pointer);
1288
1289         if dest_ptr_ty == stored_ptr_ty {
1290             ptr
1291         } else {
1292             debug!(
1293                 "type mismatch in store. \
1294                     Expected {:?}, got {:?}; inserting bitcast",
1295                 dest_ptr_ty, stored_ptr_ty
1296             );
1297             self.bitcast(ptr, stored_ptr_ty)
1298         }
1299     }
1300
1301     fn check_call<'b>(
1302         &mut self,
1303         typ: &str,
1304         llfn: &'ll Value,
1305         args: &'b [&'ll Value],
1306     ) -> Cow<'b, [&'ll Value]> {
1307         let mut fn_ty = self.cx.val_ty(llfn);
1308         // Strip off pointers
1309         while self.cx.type_kind(fn_ty) == TypeKind::Pointer {
1310             fn_ty = self.cx.element_type(fn_ty);
1311         }
1312
1313         assert!(
1314             self.cx.type_kind(fn_ty) == TypeKind::Function,
1315             "builder::{} not passed a function, but {:?}",
1316             typ,
1317             fn_ty
1318         );
1319
1320         let param_tys = self.cx.func_params_types(fn_ty);
1321
1322         let all_args_match = iter::zip(&param_tys, args.iter().map(|&v| self.val_ty(v)))
1323             .all(|(expected_ty, actual_ty)| *expected_ty == actual_ty);
1324
1325         if all_args_match {
1326             return Cow::Borrowed(args);
1327         }
1328
1329         let casted_args: Vec<_> = iter::zip(param_tys, args)
1330             .enumerate()
1331             .map(|(i, (expected_ty, &actual_val))| {
1332                 let actual_ty = self.val_ty(actual_val);
1333                 if expected_ty != actual_ty {
1334                     debug!(
1335                         "type mismatch in function call of {:?}. \
1336                             Expected {:?} for param {}, got {:?}; injecting bitcast",
1337                         llfn, expected_ty, i, actual_ty
1338                     );
1339                     self.bitcast(actual_val, expected_ty)
1340                 } else {
1341                     actual_val
1342                 }
1343             })
1344             .collect();
1345
1346         Cow::Owned(casted_args)
1347     }
1348
1349     pub fn va_arg(&mut self, list: &'ll Value, ty: &'ll Type) -> &'ll Value {
1350         unsafe { llvm::LLVMBuildVAArg(self.llbuilder, list, ty, UNNAMED) }
1351     }
1352
1353     fn call_lifetime_intrinsic(&mut self, intrinsic: &str, ptr: &'ll Value, size: Size) {
1354         let size = size.bytes();
1355         if size == 0 {
1356             return;
1357         }
1358
1359         if !self.cx().sess().emit_lifetime_markers() {
1360             return;
1361         }
1362
1363         let lifetime_intrinsic = self.cx.get_intrinsic(intrinsic);
1364
1365         let ptr = self.pointercast(ptr, self.cx.type_i8p());
1366         self.call(lifetime_intrinsic, &[self.cx.const_u64(size), ptr], None);
1367     }
1368
1369     pub(crate) fn phi(
1370         &mut self,
1371         ty: &'ll Type,
1372         vals: &[&'ll Value],
1373         bbs: &[&'ll BasicBlock],
1374     ) -> &'ll Value {
1375         assert_eq!(vals.len(), bbs.len());
1376         let phi = unsafe { llvm::LLVMBuildPhi(self.llbuilder, ty, UNNAMED) };
1377         unsafe {
1378             llvm::LLVMAddIncoming(phi, vals.as_ptr(), bbs.as_ptr(), vals.len() as c_uint);
1379             phi
1380         }
1381     }
1382
1383     fn add_incoming_to_phi(&mut self, phi: &'ll Value, val: &'ll Value, bb: &'ll BasicBlock) {
1384         unsafe {
1385             llvm::LLVMAddIncoming(phi, &val, &bb, 1 as c_uint);
1386         }
1387     }
1388
1389     fn fptoint_sat_broken_in_llvm(&self) -> bool {
1390         match self.tcx.sess.target.arch.as_str() {
1391             // FIXME - https://bugs.llvm.org/show_bug.cgi?id=50083
1392             "riscv64" => llvm_util::get_version() < (13, 0, 0),
1393             _ => false,
1394         }
1395     }
1396 }