]>
Commit | Line | Data |
---|---|---|
2b03887a | 1 | use crate::abi::FnAbiLlvmExt; |
5e7ed085 | 2 | use crate::attributes; |
9fa01778 XL |
3 | use crate::common::Funclet; |
4 | use crate::context::CodegenCx; | |
2b03887a | 5 | use crate::llvm::{self, AtomicOrdering, AtomicRmwBinOp, BasicBlock}; |
9fa01778 XL |
6 | use crate::type_::Type; |
7 | use crate::type_of::LayoutLlvmExt; | |
8 | use crate::value::Value; | |
6a06907d | 9 | use cstr::cstr; |
dfeec247 | 10 | use libc::{c_char, c_uint}; |
2b03887a | 11 | use rustc_codegen_ssa::common::{IntPredicate, RealPredicate, SynchronizationScope, TypeKind}; |
dfeec247 | 12 | use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue}; |
a1dfa0c6 | 13 | use rustc_codegen_ssa::mir::place::PlaceRef; |
dfeec247 XL |
14 | use rustc_codegen_ssa::traits::*; |
15 | use rustc_codegen_ssa::MemFlags; | |
dfeec247 XL |
16 | use rustc_data_structures::small_c_str::SmallCStr; |
17 | use rustc_hir::def_id::DefId; | |
c295e0f8 XL |
18 | use rustc_middle::ty::layout::{ |
19 | FnAbiError, FnAbiOfHelpers, FnAbiRequest, LayoutError, LayoutOfHelpers, TyAndLayout, | |
20 | }; | |
ba9703b0 | 21 | use rustc_middle::ty::{self, Ty, TyCtxt}; |
cdc7bbd5 | 22 | use rustc_span::Span; |
9c376795 | 23 | use rustc_symbol_mangling::typeid::kcfi_typeid_for_fnabi; |
c295e0f8 | 24 | use rustc_target::abi::{self, call::FnAbi, Align, Size, WrappingRange}; |
48663c56 | 25 | use rustc_target::spec::{HasTargetSpec, Target}; |
1bb2cb6e | 26 | use std::borrow::Cow; |
dc9dc135 | 27 | use std::ffi::CStr; |
cdc7bbd5 | 28 | use std::iter; |
c295e0f8 | 29 | use std::ops::Deref; |
85aaf69f | 30 | use std::ptr; |
1a4d82fc | 31 | |
32a655c1 SL |
32 | // All Builders must have an llfn associated with them |
33 | #[must_use] | |
dc9dc135 | 34 | pub struct Builder<'a, 'll, 'tcx> { |
b7449926 XL |
35 | pub llbuilder: &'ll mut llvm::Builder<'ll>, |
36 | pub cx: &'a CodegenCx<'ll, 'tcx>, | |
1a4d82fc JJ |
37 | } |
38 | ||
a2a8927a | 39 | impl Drop for Builder<'_, '_, '_> { |
32a655c1 SL |
40 | fn drop(&mut self) { |
41 | unsafe { | |
b7449926 | 42 | llvm::LLVMDisposeBuilder(&mut *(self.llbuilder as *mut _)); |
32a655c1 SL |
43 | } |
44 | } | |
45 | } | |
46 | ||
dc9dc135 | 47 | // FIXME(eddyb) use a checked constructor when they become `const fn`. |
dfeec247 | 48 | const EMPTY_C_STR: &CStr = unsafe { CStr::from_bytes_with_nul_unchecked(b"\0") }; |
dc9dc135 XL |
49 | |
50 | /// Empty string, to be used where LLVM expects an instruction name, indicating | |
51 | /// that the instruction is to be left unnamed (i.e. numbered, in textual IR). | |
52 | // FIXME(eddyb) pass `&CStr` directly to FFI once it's a thin pointer. | |
53 | const UNNAMED: *const c_char = EMPTY_C_STR.as_ptr(); | |
1a4d82fc | 54 | |
a2a8927a | 55 | impl<'ll, 'tcx> BackendTypes for Builder<'_, 'll, 'tcx> { |
a1dfa0c6 | 56 | type Value = <CodegenCx<'ll, 'tcx> as BackendTypes>::Value; |
e74abb32 | 57 | type Function = <CodegenCx<'ll, 'tcx> as BackendTypes>::Function; |
a1dfa0c6 XL |
58 | type BasicBlock = <CodegenCx<'ll, 'tcx> as BackendTypes>::BasicBlock; |
59 | type Type = <CodegenCx<'ll, 'tcx> as BackendTypes>::Type; | |
60 | type Funclet = <CodegenCx<'ll, 'tcx> as BackendTypes>::Funclet; | |
61 | ||
62 | type DIScope = <CodegenCx<'ll, 'tcx> as BackendTypes>::DIScope; | |
29967ef6 | 63 | type DILocation = <CodegenCx<'ll, 'tcx> as BackendTypes>::DILocation; |
74b04a01 | 64 | type DIVariable = <CodegenCx<'ll, 'tcx> as BackendTypes>::DIVariable; |
a1dfa0c6 XL |
65 | } |
66 | ||
ba9703b0 XL |
67 | impl abi::HasDataLayout for Builder<'_, '_, '_> { |
68 | fn data_layout(&self) -> &abi::TargetDataLayout { | |
a1dfa0c6 | 69 | self.cx.data_layout() |
83c7162d XL |
70 | } |
71 | } | |
72 | ||
a2a8927a | 73 | impl<'tcx> ty::layout::HasTyCtxt<'tcx> for Builder<'_, '_, 'tcx> { |
17df50a5 | 74 | #[inline] |
dc9dc135 | 75 | fn tcx(&self) -> TyCtxt<'tcx> { |
a1dfa0c6 XL |
76 | self.cx.tcx |
77 | } | |
78 | } | |
79 | ||
a2a8927a | 80 | impl<'tcx> ty::layout::HasParamEnv<'tcx> for Builder<'_, '_, 'tcx> { |
48663c56 XL |
81 | fn param_env(&self) -> ty::ParamEnv<'tcx> { |
82 | self.cx.param_env() | |
83 | } | |
84 | } | |
85 | ||
a2a8927a | 86 | impl HasTargetSpec for Builder<'_, '_, '_> { |
17df50a5 | 87 | #[inline] |
48663c56 | 88 | fn target_spec(&self) -> &Target { |
c295e0f8 | 89 | self.cx.target_spec() |
48663c56 XL |
90 | } |
91 | } | |
92 | ||
a2a8927a | 93 | impl<'tcx> LayoutOfHelpers<'tcx> for Builder<'_, '_, 'tcx> { |
c295e0f8 | 94 | type LayoutOfResult = TyAndLayout<'tcx>; |
a1dfa0c6 | 95 | |
c295e0f8 XL |
96 | #[inline] |
97 | fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! { | |
98 | self.cx.handle_layout_err(err, span, ty) | |
99 | } | |
100 | } | |
101 | ||
a2a8927a | 102 | impl<'tcx> FnAbiOfHelpers<'tcx> for Builder<'_, '_, 'tcx> { |
c295e0f8 XL |
103 | type FnAbiOfResult = &'tcx FnAbi<'tcx, Ty<'tcx>>; |
104 | ||
105 | #[inline] | |
106 | fn handle_fn_abi_err( | |
107 | &self, | |
108 | err: FnAbiError<'tcx>, | |
109 | span: Span, | |
110 | fn_abi_request: FnAbiRequest<'tcx>, | |
111 | ) -> ! { | |
112 | self.cx.handle_fn_abi_err(err, span, fn_abi_request) | |
a1dfa0c6 XL |
113 | } |
114 | } | |
115 | ||
a2a8927a | 116 | impl<'ll, 'tcx> Deref for Builder<'_, 'll, 'tcx> { |
a1dfa0c6 XL |
117 | type Target = CodegenCx<'ll, 'tcx>; |
118 | ||
17df50a5 | 119 | #[inline] |
a1dfa0c6 XL |
120 | fn deref(&self) -> &Self::Target { |
121 | self.cx | |
122 | } | |
123 | } | |
124 | ||
a2a8927a | 125 | impl<'ll, 'tcx> HasCodegen<'tcx> for Builder<'_, 'll, 'tcx> { |
a1dfa0c6 XL |
126 | type CodegenCx = CodegenCx<'ll, 'tcx>; |
127 | } | |
128 | ||
dc9dc135 XL |
129 | macro_rules! builder_methods_for_value_instructions { |
130 | ($($name:ident($($arg:ident),*) => $llvm_capi:ident),+ $(,)?) => { | |
131 | $(fn $name(&mut self, $($arg: &'ll Value),*) -> &'ll Value { | |
132 | unsafe { | |
133 | llvm::$llvm_capi(self.llbuilder, $($arg,)* UNNAMED) | |
134 | } | |
135 | })+ | |
136 | } | |
137 | } | |
138 | ||
a2a8927a | 139 | impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { |
17df50a5 XL |
140 | fn build(cx: &'a CodegenCx<'ll, 'tcx>, llbb: &'ll BasicBlock) -> Self { |
141 | let bx = Builder::with_cx(cx); | |
142 | unsafe { | |
143 | llvm::LLVMPositionBuilderAtEnd(bx.llbuilder, llbb); | |
144 | } | |
2c00a5a8 | 145 | bx |
32a655c1 SL |
146 | } |
147 | ||
17df50a5 XL |
148 | fn cx(&self) -> &CodegenCx<'ll, 'tcx> { |
149 | self.cx | |
32a655c1 SL |
150 | } |
151 | ||
a1dfa0c6 | 152 | fn llbb(&self) -> &'ll BasicBlock { |
dfeec247 | 153 | unsafe { llvm::LLVMGetInsertBlock(self.llbuilder) } |
32a655c1 SL |
154 | } |
155 | ||
29967ef6 XL |
156 | fn set_span(&mut self, _span: Span) {} |
157 | ||
17df50a5 | 158 | fn append_block(cx: &'a CodegenCx<'ll, 'tcx>, llfn: &'ll Value, name: &str) -> &'ll BasicBlock { |
1a4d82fc | 159 | unsafe { |
17df50a5 XL |
160 | let name = SmallCStr::new(name); |
161 | llvm::LLVMAppendBasicBlockInContext(cx.llcx, llfn, name.as_ptr()) | |
1a4d82fc JJ |
162 | } |
163 | } | |
164 | ||
17df50a5 XL |
165 | fn append_sibling_block(&mut self, name: &str) -> &'ll BasicBlock { |
166 | Self::append_block(self.cx, self.llfn(), name) | |
167 | } | |
168 | ||
5e7ed085 FG |
169 | fn switch_to_block(&mut self, llbb: Self::BasicBlock) { |
170 | *self = Self::build(self.cx, llbb) | |
17df50a5 XL |
171 | } |
172 | ||
a1dfa0c6 | 173 | fn ret_void(&mut self) { |
1a4d82fc JJ |
174 | unsafe { |
175 | llvm::LLVMBuildRetVoid(self.llbuilder); | |
176 | } | |
177 | } | |
178 | ||
a1dfa0c6 | 179 | fn ret(&mut self, v: &'ll Value) { |
1a4d82fc JJ |
180 | unsafe { |
181 | llvm::LLVMBuildRet(self.llbuilder, v); | |
182 | } | |
183 | } | |
184 | ||
a1dfa0c6 | 185 | fn br(&mut self, dest: &'ll BasicBlock) { |
1a4d82fc JJ |
186 | unsafe { |
187 | llvm::LLVMBuildBr(self.llbuilder, dest); | |
188 | } | |
189 | } | |
190 | ||
a1dfa0c6 XL |
191 | fn cond_br( |
192 | &mut self, | |
b7449926 XL |
193 | cond: &'ll Value, |
194 | then_llbb: &'ll BasicBlock, | |
195 | else_llbb: &'ll BasicBlock, | |
196 | ) { | |
1a4d82fc JJ |
197 | unsafe { |
198 | llvm::LLVMBuildCondBr(self.llbuilder, cond, then_llbb, else_llbb); | |
199 | } | |
200 | } | |
201 | ||
a1dfa0c6 XL |
202 | fn switch( |
203 | &mut self, | |
b7449926 XL |
204 | v: &'ll Value, |
205 | else_llbb: &'ll BasicBlock, | |
1b1a35ee | 206 | cases: impl ExactSizeIterator<Item = (u128, &'ll BasicBlock)>, |
532ac7d7 | 207 | ) { |
dfeec247 XL |
208 | let switch = |
209 | unsafe { llvm::LLVMBuildSwitch(self.llbuilder, v, else_llbb, cases.len() as c_uint) }; | |
532ac7d7 XL |
210 | for (on_val, dest) in cases { |
211 | let on_val = self.const_uint_big(self.val_ty(v), on_val); | |
dfeec247 | 212 | unsafe { llvm::LLVMAddCase(switch, on_val, dest) } |
1a4d82fc JJ |
213 | } |
214 | } | |
215 | ||
a1dfa0c6 XL |
216 | fn invoke( |
217 | &mut self, | |
94222f64 | 218 | llty: &'ll Type, |
2b03887a | 219 | fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, |
a1dfa0c6 XL |
220 | llfn: &'ll Value, |
221 | args: &[&'ll Value], | |
222 | then: &'ll BasicBlock, | |
223 | catch: &'ll BasicBlock, | |
224 | funclet: Option<&Funclet<'ll>>, | |
225 | ) -> &'ll Value { | |
dfeec247 | 226 | debug!("invoke {:?} with args ({:?})", llfn, args); |
1a4d82fc | 227 | |
94222f64 | 228 | let args = self.check_call("invoke", llty, llfn, args); |
9c376795 FG |
229 | let funclet_bundle = funclet.map(|funclet| funclet.bundle()); |
230 | let funclet_bundle = funclet_bundle.as_ref().map(|b| &*b.raw); | |
231 | let mut bundles = vec![funclet_bundle]; | |
232 | ||
233 | // Set KCFI operand bundle | |
234 | let is_indirect_call = unsafe { llvm::LLVMIsAFunction(llfn).is_none() }; | |
235 | let kcfi_bundle = | |
236 | if self.tcx.sess.is_sanitizer_kcfi_enabled() && let Some(fn_abi) = fn_abi && is_indirect_call { | |
237 | let kcfi_typeid = kcfi_typeid_for_fnabi(self.tcx, fn_abi); | |
238 | Some(llvm::OperandBundleDef::new("kcfi", &[self.const_u32(kcfi_typeid)])) | |
239 | } else { | |
240 | None | |
241 | }; | |
242 | if kcfi_bundle.is_some() { | |
243 | let kcfi_bundle = kcfi_bundle.as_ref().map(|b| &*b.raw); | |
244 | bundles.push(kcfi_bundle); | |
245 | } | |
7453a54e | 246 | |
9c376795 | 247 | bundles.retain(|bundle| bundle.is_some()); |
2b03887a | 248 | let invoke = unsafe { |
dfeec247 XL |
249 | llvm::LLVMRustBuildInvoke( |
250 | self.llbuilder, | |
94222f64 | 251 | llty, |
dfeec247 XL |
252 | llfn, |
253 | args.as_ptr(), | |
254 | args.len() as c_uint, | |
255 | then, | |
256 | catch, | |
9c376795 FG |
257 | bundles.as_ptr(), |
258 | bundles.len() as c_uint, | |
dfeec247 XL |
259 | UNNAMED, |
260 | ) | |
2b03887a FG |
261 | }; |
262 | if let Some(fn_abi) = fn_abi { | |
263 | fn_abi.apply_attrs_callsite(self, invoke); | |
1a4d82fc | 264 | } |
2b03887a | 265 | invoke |
1a4d82fc JJ |
266 | } |
267 | ||
a1dfa0c6 | 268 | fn unreachable(&mut self) { |
1a4d82fc JJ |
269 | unsafe { |
270 | llvm::LLVMBuildUnreachable(self.llbuilder); | |
271 | } | |
272 | } | |
273 | ||
dc9dc135 XL |
274 | builder_methods_for_value_instructions! { |
275 | add(a, b) => LLVMBuildAdd, | |
276 | fadd(a, b) => LLVMBuildFAdd, | |
277 | sub(a, b) => LLVMBuildSub, | |
278 | fsub(a, b) => LLVMBuildFSub, | |
279 | mul(a, b) => LLVMBuildMul, | |
280 | fmul(a, b) => LLVMBuildFMul, | |
281 | udiv(a, b) => LLVMBuildUDiv, | |
282 | exactudiv(a, b) => LLVMBuildExactUDiv, | |
283 | sdiv(a, b) => LLVMBuildSDiv, | |
284 | exactsdiv(a, b) => LLVMBuildExactSDiv, | |
285 | fdiv(a, b) => LLVMBuildFDiv, | |
286 | urem(a, b) => LLVMBuildURem, | |
287 | srem(a, b) => LLVMBuildSRem, | |
288 | frem(a, b) => LLVMBuildFRem, | |
289 | shl(a, b) => LLVMBuildShl, | |
290 | lshr(a, b) => LLVMBuildLShr, | |
291 | ashr(a, b) => LLVMBuildAShr, | |
292 | and(a, b) => LLVMBuildAnd, | |
293 | or(a, b) => LLVMBuildOr, | |
294 | xor(a, b) => LLVMBuildXor, | |
295 | neg(x) => LLVMBuildNeg, | |
296 | fneg(x) => LLVMBuildFNeg, | |
297 | not(x) => LLVMBuildNot, | |
298 | unchecked_sadd(x, y) => LLVMBuildNSWAdd, | |
299 | unchecked_uadd(x, y) => LLVMBuildNUWAdd, | |
300 | unchecked_ssub(x, y) => LLVMBuildNSWSub, | |
301 | unchecked_usub(x, y) => LLVMBuildNUWSub, | |
302 | unchecked_smul(x, y) => LLVMBuildNSWMul, | |
303 | unchecked_umul(x, y) => LLVMBuildNUWMul, | |
1a4d82fc JJ |
304 | } |
305 | ||
a1dfa0c6 | 306 | fn fadd_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { |
54a0048b | 307 | unsafe { |
dc9dc135 | 308 | let instr = llvm::LLVMBuildFAdd(self.llbuilder, lhs, rhs, UNNAMED); |
cdc7bbd5 | 309 | llvm::LLVMRustSetFastMath(instr); |
54a0048b SL |
310 | instr |
311 | } | |
312 | } | |
313 | ||
a1dfa0c6 | 314 | fn fsub_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { |
54a0048b | 315 | unsafe { |
dc9dc135 | 316 | let instr = llvm::LLVMBuildFSub(self.llbuilder, lhs, rhs, UNNAMED); |
cdc7bbd5 | 317 | llvm::LLVMRustSetFastMath(instr); |
54a0048b SL |
318 | instr |
319 | } | |
320 | } | |
321 | ||
a1dfa0c6 | 322 | fn fmul_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { |
54a0048b | 323 | unsafe { |
dc9dc135 | 324 | let instr = llvm::LLVMBuildFMul(self.llbuilder, lhs, rhs, UNNAMED); |
cdc7bbd5 | 325 | llvm::LLVMRustSetFastMath(instr); |
54a0048b SL |
326 | instr |
327 | } | |
328 | } | |
329 | ||
a1dfa0c6 | 330 | fn fdiv_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { |
54a0048b | 331 | unsafe { |
dc9dc135 | 332 | let instr = llvm::LLVMBuildFDiv(self.llbuilder, lhs, rhs, UNNAMED); |
cdc7bbd5 | 333 | llvm::LLVMRustSetFastMath(instr); |
54a0048b SL |
334 | instr |
335 | } | |
336 | } | |
337 | ||
a1dfa0c6 | 338 | fn frem_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { |
54a0048b | 339 | unsafe { |
dc9dc135 | 340 | let instr = llvm::LLVMBuildFRem(self.llbuilder, lhs, rhs, UNNAMED); |
cdc7bbd5 | 341 | llvm::LLVMRustSetFastMath(instr); |
54a0048b SL |
342 | instr |
343 | } | |
344 | } | |
345 | ||
a1dfa0c6 XL |
346 | fn checked_binop( |
347 | &mut self, | |
348 | oop: OverflowOp, | |
9fa01778 | 349 | ty: Ty<'_>, |
a1dfa0c6 XL |
350 | lhs: Self::Value, |
351 | rhs: Self::Value, | |
352 | ) -> (Self::Value, Self::Value) { | |
ba9703b0 | 353 | use rustc_middle::ty::{Int, Uint}; |
5869c6ff | 354 | use rustc_middle::ty::{IntTy::*, UintTy::*}; |
a1dfa0c6 | 355 | |
1b1a35ee | 356 | let new_kind = match ty.kind() { |
29967ef6 XL |
357 | Int(t @ Isize) => Int(t.normalize(self.tcx.sess.target.pointer_width)), |
358 | Uint(t @ Usize) => Uint(t.normalize(self.tcx.sess.target.pointer_width)), | |
1b1a35ee | 359 | t @ (Uint(_) | Int(_)) => t.clone(), |
dfeec247 | 360 | _ => panic!("tried to get overflow intrinsic for op applied to non-int type"), |
a1dfa0c6 XL |
361 | }; |
362 | ||
363 | let name = match oop { | |
e74abb32 | 364 | OverflowOp::Add => match new_kind { |
a1dfa0c6 XL |
365 | Int(I8) => "llvm.sadd.with.overflow.i8", |
366 | Int(I16) => "llvm.sadd.with.overflow.i16", | |
367 | Int(I32) => "llvm.sadd.with.overflow.i32", | |
368 | Int(I64) => "llvm.sadd.with.overflow.i64", | |
369 | Int(I128) => "llvm.sadd.with.overflow.i128", | |
370 | ||
371 | Uint(U8) => "llvm.uadd.with.overflow.i8", | |
372 | Uint(U16) => "llvm.uadd.with.overflow.i16", | |
373 | Uint(U32) => "llvm.uadd.with.overflow.i32", | |
374 | Uint(U64) => "llvm.uadd.with.overflow.i64", | |
375 | Uint(U128) => "llvm.uadd.with.overflow.i128", | |
376 | ||
377 | _ => unreachable!(), | |
378 | }, | |
e74abb32 | 379 | OverflowOp::Sub => match new_kind { |
a1dfa0c6 XL |
380 | Int(I8) => "llvm.ssub.with.overflow.i8", |
381 | Int(I16) => "llvm.ssub.with.overflow.i16", | |
382 | Int(I32) => "llvm.ssub.with.overflow.i32", | |
383 | Int(I64) => "llvm.ssub.with.overflow.i64", | |
384 | Int(I128) => "llvm.ssub.with.overflow.i128", | |
385 | ||
487cf647 FG |
386 | Uint(_) => { |
387 | // Emit sub and icmp instead of llvm.usub.with.overflow. LLVM considers these | |
388 | // to be the canonical form. It will attempt to reform llvm.usub.with.overflow | |
389 | // in the backend if profitable. | |
390 | let sub = self.sub(lhs, rhs); | |
391 | let cmp = self.icmp(IntPredicate::IntULT, lhs, rhs); | |
392 | return (sub, cmp); | |
393 | } | |
a1dfa0c6 XL |
394 | |
395 | _ => unreachable!(), | |
396 | }, | |
e74abb32 | 397 | OverflowOp::Mul => match new_kind { |
a1dfa0c6 XL |
398 | Int(I8) => "llvm.smul.with.overflow.i8", |
399 | Int(I16) => "llvm.smul.with.overflow.i16", | |
400 | Int(I32) => "llvm.smul.with.overflow.i32", | |
401 | Int(I64) => "llvm.smul.with.overflow.i64", | |
402 | Int(I128) => "llvm.smul.with.overflow.i128", | |
403 | ||
404 | Uint(U8) => "llvm.umul.with.overflow.i8", | |
405 | Uint(U16) => "llvm.umul.with.overflow.i16", | |
406 | Uint(U32) => "llvm.umul.with.overflow.i32", | |
407 | Uint(U64) => "llvm.umul.with.overflow.i64", | |
408 | Uint(U128) => "llvm.umul.with.overflow.i128", | |
409 | ||
410 | _ => unreachable!(), | |
411 | }, | |
412 | }; | |
413 | ||
94222f64 | 414 | let res = self.call_intrinsic(name, &[lhs, rhs]); |
dfeec247 | 415 | (self.extract_value(res, 0), self.extract_value(res, 1)) |
a1dfa0c6 XL |
416 | } |
417 | ||
1b1a35ee XL |
418 | fn from_immediate(&mut self, val: Self::Value) -> Self::Value { |
419 | if self.cx().val_ty(val) == self.cx().type_i1() { | |
420 | self.zext(val, self.cx().type_i8()) | |
421 | } else { | |
422 | val | |
423 | } | |
424 | } | |
c295e0f8 | 425 | fn to_immediate_scalar(&mut self, val: Self::Value, scalar: abi::Scalar) -> Self::Value { |
1b1a35ee XL |
426 | if scalar.is_bool() { |
427 | return self.trunc(val, self.cx().type_i1()); | |
428 | } | |
429 | val | |
430 | } | |
431 | ||
e1599b0c | 432 | fn alloca(&mut self, ty: &'ll Type, align: Align) -> &'ll Value { |
a1dfa0c6 | 433 | let mut bx = Builder::with_cx(self.cx); |
dfeec247 | 434 | bx.position_at_start(unsafe { llvm::LLVMGetFirstBasicBlock(self.llfn()) }); |
1a4d82fc | 435 | unsafe { |
2b03887a | 436 | let alloca = llvm::LLVMBuildAlloca(bx.llbuilder, ty, UNNAMED); |
a1dfa0c6 | 437 | llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint); |
cc61c64b | 438 | alloca |
1a4d82fc JJ |
439 | } |
440 | } | |
441 | ||
2b03887a | 442 | fn byte_array_alloca(&mut self, len: &'ll Value, align: Align) -> &'ll Value { |
1a4d82fc | 443 | unsafe { |
2b03887a FG |
444 | let alloca = |
445 | llvm::LLVMBuildArrayAlloca(self.llbuilder, self.cx().type_i8(), len, UNNAMED); | |
a1dfa0c6 | 446 | llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint); |
b7449926 | 447 | alloca |
1a4d82fc JJ |
448 | } |
449 | } | |
450 | ||
136023e0 | 451 | fn load(&mut self, ty: &'ll Type, ptr: &'ll Value, align: Align) -> &'ll Value { |
1a4d82fc | 452 | unsafe { |
136023e0 | 453 | let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED); |
a1dfa0c6 | 454 | llvm::LLVMSetAlignment(load, align.bytes() as c_uint); |
32a655c1 | 455 | load |
1a4d82fc JJ |
456 | } |
457 | } | |
458 | ||
136023e0 | 459 | fn volatile_load(&mut self, ty: &'ll Type, ptr: &'ll Value) -> &'ll Value { |
1a4d82fc | 460 | unsafe { |
136023e0 | 461 | let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED); |
dc9dc135 XL |
462 | llvm::LLVMSetVolatile(load, llvm::True); |
463 | load | |
1a4d82fc JJ |
464 | } |
465 | } | |
466 | ||
a1dfa0c6 XL |
467 | fn atomic_load( |
468 | &mut self, | |
136023e0 | 469 | ty: &'ll Type, |
a1dfa0c6 XL |
470 | ptr: &'ll Value, |
471 | order: rustc_codegen_ssa::common::AtomicOrdering, | |
472 | size: Size, | |
473 | ) -> &'ll Value { | |
1a4d82fc | 474 | unsafe { |
a1dfa0c6 XL |
475 | let load = llvm::LLVMRustBuildAtomicLoad( |
476 | self.llbuilder, | |
136023e0 | 477 | ty, |
a1dfa0c6 | 478 | ptr, |
dc9dc135 | 479 | UNNAMED, |
a1dfa0c6 XL |
480 | AtomicOrdering::from_generic(order), |
481 | ); | |
482 | // LLVM requires the alignment of atomic loads to be at least the size of the type. | |
483 | llvm::LLVMSetAlignment(load, size.bytes() as c_uint); | |
ff7c6d11 | 484 | load |
1a4d82fc JJ |
485 | } |
486 | } | |
487 | ||
064997fb | 488 | #[instrument(level = "trace", skip(self))] |
dfeec247 | 489 | fn load_operand(&mut self, place: PlaceRef<'tcx, &'ll Value>) -> OperandRef<'tcx, &'ll Value> { |
a1dfa0c6 XL |
490 | assert_eq!(place.llextra.is_some(), place.layout.is_unsized()); |
491 | ||
492 | if place.layout.is_zst() { | |
532ac7d7 | 493 | return OperandRef::new_zst(self, place.layout); |
a1dfa0c6 XL |
494 | } |
495 | ||
064997fb | 496 | #[instrument(level = "trace", skip(bx))] |
a1dfa0c6 XL |
497 | fn scalar_load_metadata<'a, 'll, 'tcx>( |
498 | bx: &mut Builder<'a, 'll, 'tcx>, | |
499 | load: &'ll Value, | |
c295e0f8 | 500 | scalar: abi::Scalar, |
5e7ed085 FG |
501 | layout: TyAndLayout<'tcx>, |
502 | offset: Size, | |
a1dfa0c6 | 503 | ) { |
9c376795 | 504 | if !scalar.is_uninit_valid() { |
5e7ed085 FG |
505 | bx.noundef_metadata(load); |
506 | } | |
507 | ||
04454e1e | 508 | match scalar.primitive() { |
ba9703b0 | 509 | abi::Int(..) => { |
c295e0f8 | 510 | if !scalar.is_always_valid(bx) { |
04454e1e | 511 | bx.range_metadata(load, scalar.valid_range(bx)); |
a1dfa0c6 XL |
512 | } |
513 | } | |
5e7ed085 | 514 | abi::Pointer => { |
04454e1e | 515 | if !scalar.valid_range(bx).contains(0) { |
5e7ed085 FG |
516 | bx.nonnull_metadata(load); |
517 | } | |
518 | ||
519 | if let Some(pointee) = layout.pointee_info_at(bx, offset) { | |
520 | if let Some(_) = pointee.safe { | |
521 | bx.align_metadata(load, pointee.align); | |
522 | } | |
523 | } | |
a1dfa0c6 | 524 | } |
5e7ed085 | 525 | abi::F32 | abi::F64 => {} |
a1dfa0c6 XL |
526 | } |
527 | } | |
528 | ||
529 | let val = if let Some(llextra) = place.llextra { | |
530 | OperandValue::Ref(place.llval, Some(llextra), place.align) | |
531 | } else if place.layout.is_llvm_immediate() { | |
532 | let mut const_llval = None; | |
923072b8 | 533 | let llty = place.layout.llvm_type(self); |
a1dfa0c6 XL |
534 | unsafe { |
535 | if let Some(global) = llvm::LLVMIsAGlobalVariable(place.llval) { | |
536 | if llvm::LLVMIsGlobalConstant(global) == llvm::True { | |
923072b8 FG |
537 | if let Some(init) = llvm::LLVMGetInitializer(global) { |
538 | if self.val_ty(init) == llty { | |
539 | const_llval = Some(init); | |
540 | } | |
541 | } | |
a1dfa0c6 XL |
542 | } |
543 | } | |
544 | } | |
545 | let llval = const_llval.unwrap_or_else(|| { | |
923072b8 | 546 | let load = self.load(llty, place.llval, place.align); |
c295e0f8 | 547 | if let abi::Abi::Scalar(scalar) = place.layout.abi { |
5e7ed085 | 548 | scalar_load_metadata(self, load, scalar, place.layout, Size::ZERO); |
a1dfa0c6 XL |
549 | } |
550 | load | |
551 | }); | |
1b1a35ee | 552 | OperandValue::Immediate(self.to_immediate(llval, place.layout)) |
c295e0f8 | 553 | } else if let abi::Abi::ScalarPair(a, b) = place.layout.abi { |
04454e1e | 554 | let b_offset = a.size(self).align_to(b.align(self).abi); |
94222f64 | 555 | let pair_ty = place.layout.llvm_type(self); |
a1dfa0c6 | 556 | |
5e7ed085 | 557 | let mut load = |i, scalar: abi::Scalar, layout, align, offset| { |
94222f64 | 558 | let llptr = self.struct_gep(pair_ty, place.llval, i as u64); |
136023e0 XL |
559 | let llty = place.layout.scalar_pair_element_llvm_type(self, i, false); |
560 | let load = self.load(llty, llptr, align); | |
5e7ed085 | 561 | scalar_load_metadata(self, load, scalar, layout, offset); |
1b1a35ee | 562 | self.to_immediate_scalar(load, scalar) |
a1dfa0c6 XL |
563 | }; |
564 | ||
565 | OperandValue::Pair( | |
5e7ed085 FG |
566 | load(0, a, place.layout, place.align, Size::ZERO), |
567 | load(1, b, place.layout, place.align.restrict_for_offset(b_offset), b_offset), | |
a1dfa0c6 XL |
568 | ) |
569 | } else { | |
570 | OperandValue::Ref(place.llval, None, place.align) | |
571 | }; | |
572 | ||
573 | OperandRef { val, layout: place.layout } | |
574 | } | |
575 | ||
532ac7d7 | 576 | fn write_operand_repeatedly( |
487cf647 | 577 | &mut self, |
532ac7d7 XL |
578 | cg_elem: OperandRef<'tcx, &'ll Value>, |
579 | count: u64, | |
580 | dest: PlaceRef<'tcx, &'ll Value>, | |
487cf647 | 581 | ) { |
532ac7d7 XL |
582 | let zero = self.const_usize(0); |
583 | let count = self.const_usize(count); | |
487cf647 FG |
584 | let start = dest.project_index(self, zero).llval; |
585 | let end = dest.project_index(self, count).llval; | |
532ac7d7 | 586 | |
5e7ed085 FG |
587 | let header_bb = self.append_sibling_block("repeat_loop_header"); |
588 | let body_bb = self.append_sibling_block("repeat_loop_body"); | |
589 | let next_bb = self.append_sibling_block("repeat_loop_next"); | |
590 | ||
591 | self.br(header_bb); | |
532ac7d7 | 592 | |
5e7ed085 | 593 | let mut header_bx = Self::build(self.cx, header_bb); |
532ac7d7 XL |
594 | let current = header_bx.phi(self.val_ty(start), &[start], &[self.llbb()]); |
595 | ||
596 | let keep_going = header_bx.icmp(IntPredicate::IntNE, current, end); | |
5e7ed085 | 597 | header_bx.cond_br(keep_going, body_bb, next_bb); |
532ac7d7 | 598 | |
5e7ed085 | 599 | let mut body_bx = Self::build(self.cx, body_bb); |
532ac7d7 | 600 | let align = dest.align.restrict_for_offset(dest.layout.field(self.cx(), 0).size); |
dfeec247 XL |
601 | cg_elem |
602 | .val | |
603 | .store(&mut body_bx, PlaceRef::new_sized_aligned(current, cg_elem.layout, align)); | |
a1dfa0c6 | 604 | |
94222f64 XL |
605 | let next = body_bx.inbounds_gep( |
606 | self.backend_type(cg_elem.layout), | |
607 | current, | |
608 | &[self.const_usize(1)], | |
609 | ); | |
5e7ed085 FG |
610 | body_bx.br(header_bb); |
611 | header_bx.add_incoming_to_phi(current, next, body_bb); | |
532ac7d7 | 612 | |
487cf647 | 613 | *self = Self::build(self.cx, next_bb); |
532ac7d7 | 614 | } |
1a4d82fc | 615 | |
c295e0f8 | 616 | fn range_metadata(&mut self, load: &'ll Value, range: WrappingRange) { |
29967ef6 | 617 | if self.sess().target.arch == "amdgpu" { |
94222f64 | 618 | // amdgpu/LLVM does something weird and thinks an i64 value is |
b7449926 XL |
619 | // split into a v2i32, halving the bitwidth LLVM expects, |
620 | // tripping an assertion. So, for now, just disable this | |
621 | // optimization. | |
622 | return; | |
623 | } | |
624 | ||
1a4d82fc | 625 | unsafe { |
a1dfa0c6 | 626 | let llty = self.cx.val_ty(load); |
ff7c6d11 | 627 | let v = [ |
a1dfa0c6 | 628 | self.cx.const_uint_big(llty, range.start), |
c295e0f8 | 629 | self.cx.const_uint_big(llty, range.end.wrapping_add(1)), |
ff7c6d11 | 630 | ]; |
1a4d82fc | 631 | |
dfeec247 XL |
632 | llvm::LLVMSetMetadata( |
633 | load, | |
634 | llvm::MD_range as c_uint, | |
635 | llvm::LLVMMDNodeInContext(self.cx.llcx, v.as_ptr(), v.len() as c_uint), | |
636 | ); | |
1a4d82fc | 637 | } |
1a4d82fc JJ |
638 | } |
639 | ||
a1dfa0c6 | 640 | fn nonnull_metadata(&mut self, load: &'ll Value) { |
85aaf69f | 641 | unsafe { |
dfeec247 XL |
642 | llvm::LLVMSetMetadata( |
643 | load, | |
644 | llvm::MD_nonnull as c_uint, | |
645 | llvm::LLVMMDNodeInContext(self.cx.llcx, ptr::null(), 0), | |
646 | ); | |
85aaf69f | 647 | } |
85aaf69f SL |
648 | } |
649 | ||
a1dfa0c6 | 650 | fn store(&mut self, val: &'ll Value, ptr: &'ll Value, align: Align) -> &'ll Value { |
83c7162d XL |
651 | self.store_with_flags(val, ptr, align, MemFlags::empty()) |
652 | } | |
653 | ||
a1dfa0c6 XL |
654 | fn store_with_flags( |
655 | &mut self, | |
b7449926 XL |
656 | val: &'ll Value, |
657 | ptr: &'ll Value, | |
83c7162d XL |
658 | align: Align, |
659 | flags: MemFlags, | |
b7449926 XL |
660 | ) -> &'ll Value { |
661 | debug!("Store {:?} -> {:?} ({:?})", val, ptr, flags); | |
1bb2cb6e | 662 | let ptr = self.check_store(val, ptr); |
1a4d82fc | 663 | unsafe { |
32a655c1 | 664 | let store = llvm::LLVMBuildStore(self.llbuilder, val, ptr); |
dfeec247 XL |
665 | let align = |
666 | if flags.contains(MemFlags::UNALIGNED) { 1 } else { align.bytes() as c_uint }; | |
8faf50e0 | 667 | llvm::LLVMSetAlignment(store, align); |
83c7162d XL |
668 | if flags.contains(MemFlags::VOLATILE) { |
669 | llvm::LLVMSetVolatile(store, llvm::True); | |
670 | } | |
671 | if flags.contains(MemFlags::NONTEMPORAL) { | |
672 | // According to LLVM [1] building a nontemporal store must | |
673 | // *always* point to a metadata value of the integer 1. | |
674 | // | |
136023e0 | 675 | // [1]: https://llvm.org/docs/LangRef.html#store-instruction |
a1dfa0c6 | 676 | let one = self.cx.const_i32(1); |
83c7162d XL |
677 | let node = llvm::LLVMMDNodeInContext(self.cx.llcx, &one, 1); |
678 | llvm::LLVMSetMetadata(store, llvm::MD_nontemporal as c_uint, node); | |
679 | } | |
32a655c1 | 680 | store |
1a4d82fc JJ |
681 | } |
682 | } | |
683 | ||
dfeec247 XL |
684 | fn atomic_store( |
685 | &mut self, | |
686 | val: &'ll Value, | |
687 | ptr: &'ll Value, | |
688 | order: rustc_codegen_ssa::common::AtomicOrdering, | |
689 | size: Size, | |
690 | ) { | |
b7449926 | 691 | debug!("Store {:?} -> {:?}", val, ptr); |
1bb2cb6e | 692 | let ptr = self.check_store(val, ptr); |
1a4d82fc | 693 | unsafe { |
a1dfa0c6 XL |
694 | let store = llvm::LLVMRustBuildAtomicStore( |
695 | self.llbuilder, | |
696 | val, | |
697 | ptr, | |
698 | AtomicOrdering::from_generic(order), | |
699 | ); | |
700 | // LLVM requires the alignment of atomic stores to be at least the size of the type. | |
701 | llvm::LLVMSetAlignment(store, size.bytes() as c_uint); | |
ff7c6d11 XL |
702 | } |
703 | } | |
704 | ||
94222f64 | 705 | fn gep(&mut self, ty: &'ll Type, ptr: &'ll Value, indices: &[&'ll Value]) -> &'ll Value { |
1a4d82fc | 706 | unsafe { |
94222f64 | 707 | llvm::LLVMBuildGEP2( |
dfeec247 | 708 | self.llbuilder, |
94222f64 | 709 | ty, |
dfeec247 XL |
710 | ptr, |
711 | indices.as_ptr(), | |
712 | indices.len() as c_uint, | |
713 | UNNAMED, | |
714 | ) | |
1a4d82fc JJ |
715 | } |
716 | } | |
717 | ||
94222f64 XL |
718 | fn inbounds_gep( |
719 | &mut self, | |
720 | ty: &'ll Type, | |
721 | ptr: &'ll Value, | |
722 | indices: &[&'ll Value], | |
723 | ) -> &'ll Value { | |
1a4d82fc | 724 | unsafe { |
94222f64 | 725 | llvm::LLVMBuildInBoundsGEP2( |
dfeec247 | 726 | self.llbuilder, |
94222f64 | 727 | ty, |
dfeec247 XL |
728 | ptr, |
729 | indices.as_ptr(), | |
730 | indices.len() as c_uint, | |
731 | UNNAMED, | |
732 | ) | |
1a4d82fc JJ |
733 | } |
734 | } | |
735 | ||
94222f64 | 736 | fn struct_gep(&mut self, ty: &'ll Type, ptr: &'ll Value, idx: u64) -> &'ll Value { |
532ac7d7 | 737 | assert_eq!(idx as c_uint as u64, idx); |
94222f64 | 738 | unsafe { llvm::LLVMBuildStructGEP2(self.llbuilder, ty, ptr, idx as c_uint, UNNAMED) } |
532ac7d7 XL |
739 | } |
740 | ||
1a4d82fc | 741 | /* Casts */ |
a1dfa0c6 | 742 | fn trunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { |
dfeec247 | 743 | unsafe { llvm::LLVMBuildTrunc(self.llbuilder, val, dest_ty, UNNAMED) } |
1a4d82fc JJ |
744 | } |
745 | ||
a1dfa0c6 | 746 | fn sext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { |
dfeec247 | 747 | unsafe { llvm::LLVMBuildSExt(self.llbuilder, val, dest_ty, UNNAMED) } |
1a4d82fc JJ |
748 | } |
749 | ||
f2b60f7d | 750 | fn fptoui_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { |
5099ac24 | 751 | self.fptoint_sat(false, val, dest_ty) |
f035d41b XL |
752 | } |
753 | ||
f2b60f7d | 754 | fn fptosi_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { |
5099ac24 | 755 | self.fptoint_sat(true, val, dest_ty) |
3dfed10e XL |
756 | } |
757 | ||
a1dfa0c6 | 758 | fn fptoui(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { |
cdc7bbd5 XL |
759 | // On WebAssembly the `fptoui` and `fptosi` instructions currently have |
760 | // poor codegen. The reason for this is that the corresponding wasm | |
761 | // instructions, `i32.trunc_f32_s` for example, will trap when the float | |
762 | // is out-of-bounds, infinity, or nan. This means that LLVM | |
763 | // automatically inserts control flow around `fptoui` and `fptosi` | |
764 | // because the LLVM instruction `fptoui` is defined as producing a | |
765 | // poison value, not having UB on out-of-bounds values. | |
3dfed10e | 766 | // |
cdc7bbd5 XL |
767 | // This method, however, is only used with non-saturating casts that |
768 | // have UB on out-of-bounds values. This means that it's ok if we use | |
769 | // the raw wasm instruction since out-of-bounds values can do whatever | |
770 | // we like. To ensure that LLVM picks the right instruction we choose | |
771 | // the raw wasm intrinsic functions which avoid LLVM inserting all the | |
772 | // other control flow automatically. | |
3c0e092e | 773 | if self.sess().target.is_like_wasm { |
3dfed10e XL |
774 | let src_ty = self.cx.val_ty(val); |
775 | if self.cx.type_kind(src_ty) != TypeKind::Vector { | |
776 | let float_width = self.cx.float_width(src_ty); | |
777 | let int_width = self.cx.int_width(dest_ty); | |
778 | let name = match (int_width, float_width) { | |
779 | (32, 32) => Some("llvm.wasm.trunc.unsigned.i32.f32"), | |
780 | (32, 64) => Some("llvm.wasm.trunc.unsigned.i32.f64"), | |
781 | (64, 32) => Some("llvm.wasm.trunc.unsigned.i64.f32"), | |
782 | (64, 64) => Some("llvm.wasm.trunc.unsigned.i64.f64"), | |
783 | _ => None, | |
784 | }; | |
785 | if let Some(name) = name { | |
94222f64 | 786 | return self.call_intrinsic(name, &[val]); |
3dfed10e XL |
787 | } |
788 | } | |
789 | } | |
dfeec247 | 790 | unsafe { llvm::LLVMBuildFPToUI(self.llbuilder, val, dest_ty, UNNAMED) } |
1a4d82fc JJ |
791 | } |
792 | ||
a1dfa0c6 | 793 | fn fptosi(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { |
cdc7bbd5 | 794 | // see `fptoui` above for why wasm is different here |
3c0e092e | 795 | if self.sess().target.is_like_wasm { |
3dfed10e XL |
796 | let src_ty = self.cx.val_ty(val); |
797 | if self.cx.type_kind(src_ty) != TypeKind::Vector { | |
798 | let float_width = self.cx.float_width(src_ty); | |
799 | let int_width = self.cx.int_width(dest_ty); | |
800 | let name = match (int_width, float_width) { | |
801 | (32, 32) => Some("llvm.wasm.trunc.signed.i32.f32"), | |
802 | (32, 64) => Some("llvm.wasm.trunc.signed.i32.f64"), | |
803 | (64, 32) => Some("llvm.wasm.trunc.signed.i64.f32"), | |
804 | (64, 64) => Some("llvm.wasm.trunc.signed.i64.f64"), | |
805 | _ => None, | |
806 | }; | |
807 | if let Some(name) = name { | |
94222f64 | 808 | return self.call_intrinsic(name, &[val]); |
3dfed10e XL |
809 | } |
810 | } | |
811 | } | |
dfeec247 | 812 | unsafe { llvm::LLVMBuildFPToSI(self.llbuilder, val, dest_ty, UNNAMED) } |
1a4d82fc JJ |
813 | } |
814 | ||
a1dfa0c6 | 815 | fn uitofp(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { |
dfeec247 | 816 | unsafe { llvm::LLVMBuildUIToFP(self.llbuilder, val, dest_ty, UNNAMED) } |
1a4d82fc JJ |
817 | } |
818 | ||
a1dfa0c6 | 819 | fn sitofp(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { |
dfeec247 | 820 | unsafe { llvm::LLVMBuildSIToFP(self.llbuilder, val, dest_ty, UNNAMED) } |
1a4d82fc JJ |
821 | } |
822 | ||
a1dfa0c6 | 823 | fn fptrunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { |
dfeec247 | 824 | unsafe { llvm::LLVMBuildFPTrunc(self.llbuilder, val, dest_ty, UNNAMED) } |
1a4d82fc JJ |
825 | } |
826 | ||
a1dfa0c6 | 827 | fn fpext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { |
dfeec247 | 828 | unsafe { llvm::LLVMBuildFPExt(self.llbuilder, val, dest_ty, UNNAMED) } |
1a4d82fc JJ |
829 | } |
830 | ||
a1dfa0c6 | 831 | fn ptrtoint(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { |
dfeec247 | 832 | unsafe { llvm::LLVMBuildPtrToInt(self.llbuilder, val, dest_ty, UNNAMED) } |
1a4d82fc JJ |
833 | } |
834 | ||
a1dfa0c6 | 835 | fn inttoptr(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { |
dfeec247 | 836 | unsafe { llvm::LLVMBuildIntToPtr(self.llbuilder, val, dest_ty, UNNAMED) } |
1a4d82fc JJ |
837 | } |
838 | ||
a1dfa0c6 | 839 | fn bitcast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { |
dfeec247 | 840 | unsafe { llvm::LLVMBuildBitCast(self.llbuilder, val, dest_ty, UNNAMED) } |
1a4d82fc JJ |
841 | } |
842 | ||
a1dfa0c6 | 843 | fn intcast(&mut self, val: &'ll Value, dest_ty: &'ll Type, is_signed: bool) -> &'ll Value { |
dfeec247 | 844 | unsafe { llvm::LLVMRustBuildIntCast(self.llbuilder, val, dest_ty, is_signed) } |
1a4d82fc JJ |
845 | } |
846 | ||
a1dfa0c6 | 847 | fn pointercast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { |
dfeec247 | 848 | unsafe { llvm::LLVMBuildPointerCast(self.llbuilder, val, dest_ty, UNNAMED) } |
1a4d82fc JJ |
849 | } |
850 | ||
1a4d82fc | 851 | /* Comparisons */ |
a1dfa0c6 | 852 | fn icmp(&mut self, op: IntPredicate, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { |
a1dfa0c6 | 853 | let op = llvm::IntPredicate::from_generic(op); |
dfeec247 | 854 | unsafe { llvm::LLVMBuildICmp(self.llbuilder, op as c_uint, lhs, rhs, UNNAMED) } |
1a4d82fc JJ |
855 | } |
856 | ||
a1dfa0c6 | 857 | fn fcmp(&mut self, op: RealPredicate, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { |
c295e0f8 | 858 | let op = llvm::RealPredicate::from_generic(op); |
dfeec247 | 859 | unsafe { llvm::LLVMBuildFCmp(self.llbuilder, op as c_uint, lhs, rhs, UNNAMED) } |
1a4d82fc JJ |
860 | } |
861 | ||
862 | /* Miscellaneous instructions */ | |
dfeec247 XL |
863 | fn memcpy( |
864 | &mut self, | |
865 | dst: &'ll Value, | |
866 | dst_align: Align, | |
867 | src: &'ll Value, | |
868 | src_align: Align, | |
869 | size: &'ll Value, | |
870 | flags: MemFlags, | |
871 | ) { | |
136023e0 | 872 | assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memcpy not supported"); |
a1dfa0c6 XL |
873 | let size = self.intcast(size, self.type_isize(), false); |
874 | let is_volatile = flags.contains(MemFlags::VOLATILE); | |
875 | let dst = self.pointercast(dst, self.type_i8p()); | |
876 | let src = self.pointercast(src, self.type_i8p()); | |
877 | unsafe { | |
dfeec247 XL |
878 | llvm::LLVMRustBuildMemCpy( |
879 | self.llbuilder, | |
880 | dst, | |
881 | dst_align.bytes() as c_uint, | |
882 | src, | |
883 | src_align.bytes() as c_uint, | |
884 | size, | |
885 | is_volatile, | |
886 | ); | |
a1dfa0c6 XL |
887 | } |
888 | } | |
7453a54e | 889 | |
dfeec247 XL |
890 | fn memmove( |
891 | &mut self, | |
892 | dst: &'ll Value, | |
893 | dst_align: Align, | |
894 | src: &'ll Value, | |
895 | src_align: Align, | |
896 | size: &'ll Value, | |
897 | flags: MemFlags, | |
898 | ) { | |
136023e0 | 899 | assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memmove not supported"); |
a1dfa0c6 XL |
900 | let size = self.intcast(size, self.type_isize(), false); |
901 | let is_volatile = flags.contains(MemFlags::VOLATILE); | |
902 | let dst = self.pointercast(dst, self.type_i8p()); | |
903 | let src = self.pointercast(src, self.type_i8p()); | |
1a4d82fc | 904 | unsafe { |
dfeec247 XL |
905 | llvm::LLVMRustBuildMemMove( |
906 | self.llbuilder, | |
907 | dst, | |
908 | dst_align.bytes() as c_uint, | |
909 | src, | |
910 | src_align.bytes() as c_uint, | |
911 | size, | |
912 | is_volatile, | |
913 | ); | |
1a4d82fc JJ |
914 | } |
915 | } | |
916 | ||
a1dfa0c6 XL |
917 | fn memset( |
918 | &mut self, | |
919 | ptr: &'ll Value, | |
920 | fill_byte: &'ll Value, | |
921 | size: &'ll Value, | |
922 | align: Align, | |
923 | flags: MemFlags, | |
924 | ) { | |
74b04a01 | 925 | let is_volatile = flags.contains(MemFlags::VOLATILE); |
a1dfa0c6 | 926 | let ptr = self.pointercast(ptr, self.type_i8p()); |
74b04a01 XL |
927 | unsafe { |
928 | llvm::LLVMRustBuildMemSet( | |
929 | self.llbuilder, | |
930 | ptr, | |
931 | align.bytes() as c_uint, | |
932 | fill_byte, | |
933 | size, | |
934 | is_volatile, | |
935 | ); | |
936 | } | |
a1dfa0c6 XL |
937 | } |
938 | ||
a1dfa0c6 | 939 | fn select( |
dfeec247 XL |
940 | &mut self, |
941 | cond: &'ll Value, | |
b7449926 XL |
942 | then_val: &'ll Value, |
943 | else_val: &'ll Value, | |
944 | ) -> &'ll Value { | |
dfeec247 | 945 | unsafe { llvm::LLVMBuildSelect(self.llbuilder, cond, then_val, else_val, UNNAMED) } |
1a4d82fc JJ |
946 | } |
947 | ||
a1dfa0c6 | 948 | fn va_arg(&mut self, list: &'ll Value, ty: &'ll Type) -> &'ll Value { |
dfeec247 | 949 | unsafe { llvm::LLVMBuildVAArg(self.llbuilder, list, ty, UNNAMED) } |
1a4d82fc JJ |
950 | } |
951 | ||
a1dfa0c6 | 952 | fn extract_element(&mut self, vec: &'ll Value, idx: &'ll Value) -> &'ll Value { |
dfeec247 | 953 | unsafe { llvm::LLVMBuildExtractElement(self.llbuilder, vec, idx, UNNAMED) } |
1a4d82fc JJ |
954 | } |
955 | ||
a1dfa0c6 | 956 | fn vector_splat(&mut self, num_elts: usize, elt: &'ll Value) -> &'ll Value { |
1a4d82fc | 957 | unsafe { |
a1dfa0c6 XL |
958 | let elt_ty = self.cx.val_ty(elt); |
959 | let undef = llvm::LLVMGetUndef(self.type_vector(elt_ty, num_elts as u64)); | |
960 | let vec = self.insert_element(undef, elt, self.cx.const_i32(0)); | |
961 | let vec_i32_ty = self.type_vector(self.type_i32(), num_elts as u64); | |
962 | self.shuffle_vector(vec, undef, self.const_null(vec_i32_ty)) | |
1a4d82fc JJ |
963 | } |
964 | } | |
965 | ||
a1dfa0c6 | 966 | fn extract_value(&mut self, agg_val: &'ll Value, idx: u64) -> &'ll Value { |
ff7c6d11 | 967 | assert_eq!(idx as c_uint as u64, idx); |
dfeec247 | 968 | unsafe { llvm::LLVMBuildExtractValue(self.llbuilder, agg_val, idx as c_uint, UNNAMED) } |
1a4d82fc JJ |
969 | } |
970 | ||
dfeec247 | 971 | fn insert_value(&mut self, agg_val: &'ll Value, elt: &'ll Value, idx: u64) -> &'ll Value { |
ff7c6d11 | 972 | assert_eq!(idx as c_uint as u64, idx); |
dfeec247 | 973 | unsafe { llvm::LLVMBuildInsertValue(self.llbuilder, agg_val, elt, idx as c_uint, UNNAMED) } |
1a4d82fc JJ |
974 | } |
975 | ||
5099ac24 | 976 | fn set_personality_fn(&mut self, personality: &'ll Value) { |
1a4d82fc | 977 | unsafe { |
5099ac24 | 978 | llvm::LLVMSetPersonalityFn(self.llfn(), personality); |
c1a9b12d SL |
979 | } |
980 | } | |
981 | ||
9c376795 FG |
982 | fn cleanup_landing_pad(&mut self, pers_fn: &'ll Value) -> (&'ll Value, &'ll Value) { |
983 | let ty = self.type_struct(&[self.type_i8p(), self.type_i32()], false); | |
5099ac24 | 984 | let landing_pad = self.landing_pad(ty, pers_fn, 1 /* FIXME should this be 0? */); |
1a4d82fc JJ |
985 | unsafe { |
986 | llvm::LLVMSetCleanup(landing_pad, llvm::True); | |
987 | } | |
9c376795 | 988 | (self.extract_value(landing_pad, 0), self.extract_value(landing_pad, 1)) |
1a4d82fc JJ |
989 | } |
990 | ||
9c376795 FG |
991 | fn resume(&mut self, exn0: &'ll Value, exn1: &'ll Value) { |
992 | let ty = self.type_struct(&[self.type_i8p(), self.type_i32()], false); | |
993 | let mut exn = self.const_undef(ty); | |
994 | exn = self.insert_value(exn, exn0, 0); | |
995 | exn = self.insert_value(exn, exn1, 1); | |
5099ac24 FG |
996 | unsafe { |
997 | llvm::LLVMBuildResume(self.llbuilder, exn); | |
998 | } | |
1a4d82fc JJ |
999 | } |
1000 | ||
dfeec247 | 1001 | fn cleanup_pad(&mut self, parent: Option<&'ll Value>, args: &[&'ll Value]) -> Funclet<'ll> { |
6a06907d | 1002 | let name = cstr!("cleanuppad"); |
7453a54e | 1003 | let ret = unsafe { |
dfeec247 XL |
1004 | llvm::LLVMRustBuildCleanupPad( |
1005 | self.llbuilder, | |
1006 | parent, | |
1007 | args.len() as c_uint, | |
1008 | args.as_ptr(), | |
1009 | name.as_ptr(), | |
1010 | ) | |
7453a54e | 1011 | }; |
a1dfa0c6 | 1012 | Funclet::new(ret.expect("LLVM does not have support for cleanuppad")) |
7453a54e SL |
1013 | } |
1014 | ||
5099ac24 FG |
1015 | fn cleanup_ret(&mut self, funclet: &Funclet<'ll>, unwind: Option<&'ll BasicBlock>) { |
1016 | unsafe { | |
1017 | llvm::LLVMRustBuildCleanupRet(self.llbuilder, funclet.cleanuppad(), unwind) | |
1018 | .expect("LLVM does not have support for cleanupret"); | |
1019 | } | |
7453a54e SL |
1020 | } |
1021 | ||
dfeec247 | 1022 | fn catch_pad(&mut self, parent: &'ll Value, args: &[&'ll Value]) -> Funclet<'ll> { |
6a06907d | 1023 | let name = cstr!("catchpad"); |
7453a54e | 1024 | let ret = unsafe { |
dfeec247 XL |
1025 | llvm::LLVMRustBuildCatchPad( |
1026 | self.llbuilder, | |
1027 | parent, | |
1028 | args.len() as c_uint, | |
1029 | args.as_ptr(), | |
1030 | name.as_ptr(), | |
1031 | ) | |
7453a54e | 1032 | }; |
a1dfa0c6 | 1033 | Funclet::new(ret.expect("LLVM does not have support for catchpad")) |
7453a54e SL |
1034 | } |
1035 | ||
a1dfa0c6 XL |
1036 | fn catch_switch( |
1037 | &mut self, | |
b7449926 XL |
1038 | parent: Option<&'ll Value>, |
1039 | unwind: Option<&'ll BasicBlock>, | |
5099ac24 | 1040 | handlers: &[&'ll BasicBlock], |
b7449926 | 1041 | ) -> &'ll Value { |
6a06907d | 1042 | let name = cstr!("catchswitch"); |
7453a54e | 1043 | let ret = unsafe { |
dfeec247 XL |
1044 | llvm::LLVMRustBuildCatchSwitch( |
1045 | self.llbuilder, | |
1046 | parent, | |
1047 | unwind, | |
5099ac24 | 1048 | handlers.len() as c_uint, |
dfeec247 XL |
1049 | name.as_ptr(), |
1050 | ) | |
7453a54e | 1051 | }; |
5099ac24 FG |
1052 | let ret = ret.expect("LLVM does not have support for catchswitch"); |
1053 | for handler in handlers { | |
1054 | unsafe { | |
1055 | llvm::LLVMRustAddHandler(ret, handler); | |
1056 | } | |
7453a54e | 1057 | } |
5099ac24 | 1058 | ret |
7453a54e SL |
1059 | } |
1060 | ||
1a4d82fc | 1061 | // Atomic Operations |
a1dfa0c6 XL |
1062 | fn atomic_cmpxchg( |
1063 | &mut self, | |
b7449926 XL |
1064 | dst: &'ll Value, |
1065 | cmp: &'ll Value, | |
1066 | src: &'ll Value, | |
f2b60f7d | 1067 | order: rustc_codegen_ssa::common::AtomicOrdering, |
a1dfa0c6 XL |
1068 | failure_order: rustc_codegen_ssa::common::AtomicOrdering, |
1069 | weak: bool, | |
b7449926 | 1070 | ) -> &'ll Value { |
a1dfa0c6 XL |
1071 | let weak = if weak { llvm::True } else { llvm::False }; |
1072 | unsafe { | |
2b03887a | 1073 | let value = llvm::LLVMBuildAtomicCmpXchg( |
a1dfa0c6 XL |
1074 | self.llbuilder, |
1075 | dst, | |
1076 | cmp, | |
1077 | src, | |
1078 | AtomicOrdering::from_generic(order), | |
1079 | AtomicOrdering::from_generic(failure_order), | |
2b03887a FG |
1080 | llvm::False, // SingleThreaded |
1081 | ); | |
1082 | llvm::LLVMSetWeak(value, weak); | |
1083 | value | |
1a4d82fc JJ |
1084 | } |
1085 | } | |
a1dfa0c6 XL |
1086 | fn atomic_rmw( |
1087 | &mut self, | |
1088 | op: rustc_codegen_ssa::common::AtomicRmwBinOp, | |
b7449926 XL |
1089 | dst: &'ll Value, |
1090 | src: &'ll Value, | |
a1dfa0c6 | 1091 | order: rustc_codegen_ssa::common::AtomicOrdering, |
b7449926 | 1092 | ) -> &'ll Value { |
1a4d82fc | 1093 | unsafe { |
a1dfa0c6 XL |
1094 | llvm::LLVMBuildAtomicRMW( |
1095 | self.llbuilder, | |
1096 | AtomicRmwBinOp::from_generic(op), | |
1097 | dst, | |
1098 | src, | |
1099 | AtomicOrdering::from_generic(order), | |
2b03887a | 1100 | llvm::False, // SingleThreaded |
dfeec247 | 1101 | ) |
1a4d82fc JJ |
1102 | } |
1103 | } | |
1104 | ||
a1dfa0c6 XL |
1105 | fn atomic_fence( |
1106 | &mut self, | |
1107 | order: rustc_codegen_ssa::common::AtomicOrdering, | |
2b03887a | 1108 | scope: SynchronizationScope, |
a1dfa0c6 | 1109 | ) { |
2b03887a FG |
1110 | let single_threaded = match scope { |
1111 | SynchronizationScope::SingleThread => llvm::True, | |
1112 | SynchronizationScope::CrossThread => llvm::False, | |
1113 | }; | |
1a4d82fc | 1114 | unsafe { |
2b03887a | 1115 | llvm::LLVMBuildFence( |
a1dfa0c6 XL |
1116 | self.llbuilder, |
1117 | AtomicOrdering::from_generic(order), | |
2b03887a FG |
1118 | single_threaded, |
1119 | UNNAMED, | |
a1dfa0c6 | 1120 | ); |
1a4d82fc JJ |
1121 | } |
1122 | } | |
a7813a04 | 1123 | |
532ac7d7 | 1124 | fn set_invariant_load(&mut self, load: &'ll Value) { |
32a655c1 | 1125 | unsafe { |
dfeec247 XL |
1126 | llvm::LLVMSetMetadata( |
1127 | load, | |
1128 | llvm::MD_invariant_load as c_uint, | |
1129 | llvm::LLVMMDNodeInContext(self.cx.llcx, ptr::null(), 0), | |
1130 | ); | |
32a655c1 SL |
1131 | } |
1132 | } | |
1133 | ||
532ac7d7 | 1134 | fn lifetime_start(&mut self, ptr: &'ll Value, size: Size) { |
74b04a01 | 1135 | self.call_lifetime_intrinsic("llvm.lifetime.start.p0i8", ptr, size); |
532ac7d7 XL |
1136 | } |
1137 | ||
1138 | fn lifetime_end(&mut self, ptr: &'ll Value, size: Size) { | |
74b04a01 | 1139 | self.call_lifetime_intrinsic("llvm.lifetime.end.p0i8", ptr, size); |
532ac7d7 XL |
1140 | } |
1141 | ||
f035d41b XL |
1142 | fn instrprof_increment( |
1143 | &mut self, | |
1144 | fn_name: &'ll Value, | |
1145 | hash: &'ll Value, | |
1146 | num_counters: &'ll Value, | |
1147 | index: &'ll Value, | |
3dfed10e | 1148 | ) { |
f035d41b XL |
1149 | debug!( |
1150 | "instrprof_increment() with args ({:?}, {:?}, {:?}, {:?})", | |
1151 | fn_name, hash, num_counters, index | |
1152 | ); | |
1153 | ||
3dfed10e | 1154 | let llfn = unsafe { llvm::LLVMRustGetInstrProfIncrementIntrinsic(self.cx().llmod) }; |
94222f64 XL |
1155 | let llty = self.cx.type_func( |
1156 | &[self.cx.type_i8p(), self.cx.type_i64(), self.cx.type_i32(), self.cx.type_i32()], | |
1157 | self.cx.type_void(), | |
1158 | ); | |
f035d41b | 1159 | let args = &[fn_name, hash, num_counters, index]; |
94222f64 | 1160 | let args = self.check_call("call", llty, llfn, args); |
f035d41b XL |
1161 | |
1162 | unsafe { | |
3dfed10e | 1163 | let _ = llvm::LLVMRustBuildCall( |
f035d41b | 1164 | self.llbuilder, |
94222f64 | 1165 | llty, |
f035d41b XL |
1166 | llfn, |
1167 | args.as_ptr() as *const &llvm::Value, | |
1168 | args.len() as c_uint, | |
9c376795 FG |
1169 | [].as_ptr(), |
1170 | 0 as c_uint, | |
3dfed10e | 1171 | ); |
f035d41b XL |
1172 | } |
1173 | } | |
1174 | ||
532ac7d7 XL |
1175 | fn call( |
1176 | &mut self, | |
94222f64 | 1177 | llty: &'ll Type, |
2b03887a | 1178 | fn_abi: Option<&FnAbi<'tcx, Ty<'tcx>>>, |
532ac7d7 XL |
1179 | llfn: &'ll Value, |
1180 | args: &[&'ll Value], | |
1181 | funclet: Option<&Funclet<'ll>>, | |
1182 | ) -> &'ll Value { | |
dfeec247 | 1183 | debug!("call {:?} with args ({:?})", llfn, args); |
532ac7d7 | 1184 | |
94222f64 | 1185 | let args = self.check_call("call", llty, llfn, args); |
9c376795 FG |
1186 | let funclet_bundle = funclet.map(|funclet| funclet.bundle()); |
1187 | let funclet_bundle = funclet_bundle.as_ref().map(|b| &*b.raw); | |
1188 | let mut bundles = vec![funclet_bundle]; | |
1189 | ||
1190 | // Set KCFI operand bundle | |
1191 | let is_indirect_call = unsafe { llvm::LLVMIsAFunction(llfn).is_none() }; | |
1192 | let kcfi_bundle = | |
1193 | if self.tcx.sess.is_sanitizer_kcfi_enabled() && fn_abi.is_some() && is_indirect_call { | |
1194 | let kcfi_typeid = kcfi_typeid_for_fnabi(self.tcx, fn_abi.unwrap()); | |
1195 | Some(llvm::OperandBundleDef::new("kcfi", &[self.const_u32(kcfi_typeid)])) | |
1196 | } else { | |
1197 | None | |
1198 | }; | |
1199 | if kcfi_bundle.is_some() { | |
1200 | let kcfi_bundle = kcfi_bundle.as_ref().map(|b| &*b.raw); | |
1201 | bundles.push(kcfi_bundle); | |
1202 | } | |
532ac7d7 | 1203 | |
9c376795 | 1204 | bundles.retain(|bundle| bundle.is_some()); |
2b03887a | 1205 | let call = unsafe { |
532ac7d7 XL |
1206 | llvm::LLVMRustBuildCall( |
1207 | self.llbuilder, | |
94222f64 | 1208 | llty, |
532ac7d7 XL |
1209 | llfn, |
1210 | args.as_ptr() as *const &llvm::Value, | |
1211 | args.len() as c_uint, | |
9c376795 FG |
1212 | bundles.as_ptr(), |
1213 | bundles.len() as c_uint, | |
532ac7d7 | 1214 | ) |
2b03887a FG |
1215 | }; |
1216 | if let Some(fn_abi) = fn_abi { | |
1217 | fn_abi.apply_attrs_callsite(self, call); | |
32a655c1 | 1218 | } |
2b03887a | 1219 | call |
32a655c1 SL |
1220 | } |
1221 | ||
532ac7d7 | 1222 | fn zext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { |
dfeec247 | 1223 | unsafe { llvm::LLVMBuildZExt(self.llbuilder, val, dest_ty, UNNAMED) } |
8bb4bdeb XL |
1224 | } |
1225 | ||
5e7ed085 FG |
1226 | fn do_not_inline(&mut self, llret: &'ll Value) { |
1227 | let noinline = llvm::AttributeKind::NoInline.create_attr(self.llcx); | |
1228 | attributes::apply_to_callsite(llret, llvm::AttributePlace::Function, &[noinline]); | |
532ac7d7 XL |
1229 | } |
1230 | } | |
1231 | ||
a2a8927a | 1232 | impl<'ll> StaticBuilderMethods for Builder<'_, 'll, '_> { |
dc9dc135 | 1233 | fn get_static(&mut self, def_id: DefId) -> &'ll Value { |
532ac7d7 XL |
1234 | // Forward to the `get_static` method of `CodegenCx` |
1235 | self.cx().get_static(def_id) | |
1236 | } | |
532ac7d7 XL |
1237 | } |
1238 | ||
a2a8927a | 1239 | impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { |
17df50a5 XL |
1240 | fn with_cx(cx: &'a CodegenCx<'ll, 'tcx>) -> Self { |
1241 | // Create a fresh builder from the crate context. | |
1242 | let llbuilder = unsafe { llvm::LLVMCreateBuilderInContext(cx.llcx) }; | |
1243 | Builder { llbuilder, cx } | |
1244 | } | |
1245 | ||
532ac7d7 | 1246 | pub fn llfn(&self) -> &'ll Value { |
dfeec247 | 1247 | unsafe { llvm::LLVMGetBasicBlockParent(self.llbb()) } |
532ac7d7 XL |
1248 | } |
1249 | ||
532ac7d7 XL |
1250 | fn position_at_start(&mut self, llbb: &'ll BasicBlock) { |
1251 | unsafe { | |
1252 | llvm::LLVMRustPositionBuilderAtStart(self.llbuilder, llbb); | |
1253 | } | |
1254 | } | |
1255 | ||
5e7ed085 FG |
1256 | fn align_metadata(&mut self, load: &'ll Value, align: Align) { |
1257 | unsafe { | |
1258 | let v = [self.cx.const_u64(align.bytes())]; | |
1259 | ||
1260 | llvm::LLVMSetMetadata( | |
1261 | load, | |
1262 | llvm::MD_align as c_uint, | |
1263 | llvm::LLVMMDNodeInContext(self.cx.llcx, v.as_ptr(), v.len() as c_uint), | |
1264 | ); | |
1265 | } | |
1266 | } | |
1267 | ||
1268 | fn noundef_metadata(&mut self, load: &'ll Value) { | |
1269 | unsafe { | |
1270 | llvm::LLVMSetMetadata( | |
1271 | load, | |
1272 | llvm::MD_noundef as c_uint, | |
1273 | llvm::LLVMMDNodeInContext(self.cx.llcx, ptr::null(), 0), | |
1274 | ); | |
1275 | } | |
1276 | } | |
1277 | ||
532ac7d7 | 1278 | pub fn minnum(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { |
532ac7d7 XL |
1279 | unsafe { llvm::LLVMRustBuildMinNum(self.llbuilder, lhs, rhs) } |
1280 | } | |
1281 | ||
1282 | pub fn maxnum(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { | |
532ac7d7 XL |
1283 | unsafe { llvm::LLVMRustBuildMaxNum(self.llbuilder, lhs, rhs) } |
1284 | } | |
1285 | ||
1286 | pub fn insert_element( | |
dfeec247 XL |
1287 | &mut self, |
1288 | vec: &'ll Value, | |
532ac7d7 XL |
1289 | elt: &'ll Value, |
1290 | idx: &'ll Value, | |
1291 | ) -> &'ll Value { | |
dfeec247 | 1292 | unsafe { llvm::LLVMBuildInsertElement(self.llbuilder, vec, elt, idx, UNNAMED) } |
532ac7d7 XL |
1293 | } |
1294 | ||
1295 | pub fn shuffle_vector( | |
1296 | &mut self, | |
1297 | v1: &'ll Value, | |
1298 | v2: &'ll Value, | |
1299 | mask: &'ll Value, | |
1300 | ) -> &'ll Value { | |
dfeec247 | 1301 | unsafe { llvm::LLVMBuildShuffleVector(self.llbuilder, v1, v2, mask, UNNAMED) } |
532ac7d7 XL |
1302 | } |
1303 | ||
416331ca XL |
1304 | pub fn vector_reduce_fadd(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { |
1305 | unsafe { llvm::LLVMRustBuildVectorReduceFAdd(self.llbuilder, acc, src) } | |
1306 | } | |
1307 | pub fn vector_reduce_fmul(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { | |
1308 | unsafe { llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src) } | |
1309 | } | |
532ac7d7 | 1310 | pub fn vector_reduce_fadd_fast(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { |
532ac7d7 | 1311 | unsafe { |
532ac7d7 | 1312 | let instr = llvm::LLVMRustBuildVectorReduceFAdd(self.llbuilder, acc, src); |
cdc7bbd5 | 1313 | llvm::LLVMRustSetFastMath(instr); |
532ac7d7 XL |
1314 | instr |
1315 | } | |
1316 | } | |
1317 | pub fn vector_reduce_fmul_fast(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { | |
532ac7d7 | 1318 | unsafe { |
532ac7d7 | 1319 | let instr = llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src); |
cdc7bbd5 | 1320 | llvm::LLVMRustSetFastMath(instr); |
532ac7d7 XL |
1321 | instr |
1322 | } | |
1323 | } | |
1324 | pub fn vector_reduce_add(&mut self, src: &'ll Value) -> &'ll Value { | |
532ac7d7 XL |
1325 | unsafe { llvm::LLVMRustBuildVectorReduceAdd(self.llbuilder, src) } |
1326 | } | |
1327 | pub fn vector_reduce_mul(&mut self, src: &'ll Value) -> &'ll Value { | |
532ac7d7 XL |
1328 | unsafe { llvm::LLVMRustBuildVectorReduceMul(self.llbuilder, src) } |
1329 | } | |
1330 | pub fn vector_reduce_and(&mut self, src: &'ll Value) -> &'ll Value { | |
532ac7d7 XL |
1331 | unsafe { llvm::LLVMRustBuildVectorReduceAnd(self.llbuilder, src) } |
1332 | } | |
1333 | pub fn vector_reduce_or(&mut self, src: &'ll Value) -> &'ll Value { | |
532ac7d7 XL |
1334 | unsafe { llvm::LLVMRustBuildVectorReduceOr(self.llbuilder, src) } |
1335 | } | |
1336 | pub fn vector_reduce_xor(&mut self, src: &'ll Value) -> &'ll Value { | |
532ac7d7 XL |
1337 | unsafe { llvm::LLVMRustBuildVectorReduceXor(self.llbuilder, src) } |
1338 | } | |
1339 | pub fn vector_reduce_fmin(&mut self, src: &'ll Value) -> &'ll Value { | |
dfeec247 XL |
1340 | unsafe { |
1341 | llvm::LLVMRustBuildVectorReduceFMin(self.llbuilder, src, /*NoNaNs:*/ false) | |
1342 | } | |
532ac7d7 XL |
1343 | } |
1344 | pub fn vector_reduce_fmax(&mut self, src: &'ll Value) -> &'ll Value { | |
dfeec247 XL |
1345 | unsafe { |
1346 | llvm::LLVMRustBuildVectorReduceFMax(self.llbuilder, src, /*NoNaNs:*/ false) | |
1347 | } | |
532ac7d7 XL |
1348 | } |
1349 | pub fn vector_reduce_fmin_fast(&mut self, src: &'ll Value) -> &'ll Value { | |
532ac7d7 | 1350 | unsafe { |
dfeec247 XL |
1351 | let instr = |
1352 | llvm::LLVMRustBuildVectorReduceFMin(self.llbuilder, src, /*NoNaNs:*/ true); | |
cdc7bbd5 | 1353 | llvm::LLVMRustSetFastMath(instr); |
532ac7d7 XL |
1354 | instr |
1355 | } | |
1356 | } | |
1357 | pub fn vector_reduce_fmax_fast(&mut self, src: &'ll Value) -> &'ll Value { | |
532ac7d7 | 1358 | unsafe { |
dfeec247 XL |
1359 | let instr = |
1360 | llvm::LLVMRustBuildVectorReduceFMax(self.llbuilder, src, /*NoNaNs:*/ true); | |
cdc7bbd5 | 1361 | llvm::LLVMRustSetFastMath(instr); |
532ac7d7 XL |
1362 | instr |
1363 | } | |
1364 | } | |
1365 | pub fn vector_reduce_min(&mut self, src: &'ll Value, is_signed: bool) -> &'ll Value { | |
532ac7d7 XL |
1366 | unsafe { llvm::LLVMRustBuildVectorReduceMin(self.llbuilder, src, is_signed) } |
1367 | } | |
1368 | pub fn vector_reduce_max(&mut self, src: &'ll Value, is_signed: bool) -> &'ll Value { | |
532ac7d7 XL |
1369 | unsafe { llvm::LLVMRustBuildVectorReduceMax(self.llbuilder, src, is_signed) } |
1370 | } | |
1371 | ||
1372 | pub fn add_clause(&mut self, landing_pad: &'ll Value, clause: &'ll Value) { | |
1373 | unsafe { | |
1374 | llvm::LLVMAddClause(landing_pad, clause); | |
1375 | } | |
1376 | } | |
1377 | ||
1378 | pub fn catch_ret(&mut self, funclet: &Funclet<'ll>, unwind: &'ll BasicBlock) -> &'ll Value { | |
dfeec247 XL |
1379 | let ret = |
1380 | unsafe { llvm::LLVMRustBuildCatchRet(self.llbuilder, funclet.cleanuppad(), unwind) }; | |
532ac7d7 XL |
1381 | ret.expect("LLVM does not have support for catchret") |
1382 | } | |
1383 | ||
dc9dc135 | 1384 | fn check_store(&mut self, val: &'ll Value, ptr: &'ll Value) -> &'ll Value { |
a1dfa0c6 XL |
1385 | let dest_ptr_ty = self.cx.val_ty(ptr); |
1386 | let stored_ty = self.cx.val_ty(val); | |
1387 | let stored_ptr_ty = self.cx.type_ptr_to(stored_ty); | |
1bb2cb6e | 1388 | |
a1dfa0c6 | 1389 | assert_eq!(self.cx.type_kind(dest_ptr_ty), TypeKind::Pointer); |
1bb2cb6e SL |
1390 | |
1391 | if dest_ptr_ty == stored_ptr_ty { | |
1392 | ptr | |
1393 | } else { | |
dfeec247 XL |
1394 | debug!( |
1395 | "type mismatch in store. \ | |
1bb2cb6e | 1396 | Expected {:?}, got {:?}; inserting bitcast", |
dfeec247 XL |
1397 | dest_ptr_ty, stored_ptr_ty |
1398 | ); | |
1bb2cb6e SL |
1399 | self.bitcast(ptr, stored_ptr_ty) |
1400 | } | |
1401 | } | |
1402 | ||
dfeec247 XL |
1403 | fn check_call<'b>( |
1404 | &mut self, | |
1405 | typ: &str, | |
94222f64 | 1406 | fn_ty: &'ll Type, |
dfeec247 XL |
1407 | llfn: &'ll Value, |
1408 | args: &'b [&'ll Value], | |
1409 | ) -> Cow<'b, [&'ll Value]> { | |
dfeec247 XL |
1410 | assert!( |
1411 | self.cx.type_kind(fn_ty) == TypeKind::Function, | |
1412 | "builder::{} not passed a function, but {:?}", | |
1413 | typ, | |
1414 | fn_ty | |
1415 | ); | |
a7813a04 | 1416 | |
a1dfa0c6 | 1417 | let param_tys = self.cx.func_params_types(fn_ty); |
a7813a04 | 1418 | |
cdc7bbd5 | 1419 | let all_args_match = iter::zip(¶m_tys, args.iter().map(|&v| self.val_ty(v))) |
1bb2cb6e SL |
1420 | .all(|(expected_ty, actual_ty)| *expected_ty == actual_ty); |
1421 | ||
1422 | if all_args_match { | |
1423 | return Cow::Borrowed(args); | |
1424 | } | |
1425 | ||
cdc7bbd5 | 1426 | let casted_args: Vec<_> = iter::zip(param_tys, args) |
1bb2cb6e SL |
1427 | .enumerate() |
1428 | .map(|(i, (expected_ty, &actual_val))| { | |
a1dfa0c6 | 1429 | let actual_ty = self.val_ty(actual_val); |
1bb2cb6e | 1430 | if expected_ty != actual_ty { |
dfeec247 XL |
1431 | debug!( |
1432 | "type mismatch in function call of {:?}. \ | |
1bb2cb6e | 1433 | Expected {:?} for param {}, got {:?}; injecting bitcast", |
dfeec247 XL |
1434 | llfn, expected_ty, i, actual_ty |
1435 | ); | |
1bb2cb6e SL |
1436 | self.bitcast(actual_val, expected_ty) |
1437 | } else { | |
1438 | actual_val | |
1439 | } | |
1440 | }) | |
1441 | .collect(); | |
a7813a04 | 1442 | |
0bf4aa26 | 1443 | Cow::Owned(casted_args) |
a7813a04 | 1444 | } |
ff7c6d11 | 1445 | |
532ac7d7 | 1446 | pub fn va_arg(&mut self, list: &'ll Value, ty: &'ll Type) -> &'ll Value { |
dfeec247 | 1447 | unsafe { llvm::LLVMBuildVAArg(self.llbuilder, list, ty, UNNAMED) } |
a1dfa0c6 XL |
1448 | } |
1449 | ||
923072b8 | 1450 | pub(crate) fn call_intrinsic(&mut self, intrinsic: &str, args: &[&'ll Value]) -> &'ll Value { |
94222f64 | 1451 | let (ty, f) = self.cx.get_intrinsic(intrinsic); |
2b03887a | 1452 | self.call(ty, None, f, args, None) |
94222f64 XL |
1453 | } |
1454 | ||
a1dfa0c6 | 1455 | fn call_lifetime_intrinsic(&mut self, intrinsic: &str, ptr: &'ll Value, size: Size) { |
74b04a01 XL |
1456 | let size = size.bytes(); |
1457 | if size == 0 { | |
ff7c6d11 XL |
1458 | return; |
1459 | } | |
1460 | ||
f9f354fc | 1461 | if !self.cx().sess().emit_lifetime_markers() { |
ff7c6d11 XL |
1462 | return; |
1463 | } | |
1464 | ||
a1dfa0c6 | 1465 | let ptr = self.pointercast(ptr, self.cx.type_i8p()); |
94222f64 | 1466 | self.call_intrinsic(intrinsic, &[self.cx.const_u64(size), ptr]); |
ff7c6d11 | 1467 | } |
532ac7d7 | 1468 | |
3dfed10e XL |
1469 | pub(crate) fn phi( |
1470 | &mut self, | |
1471 | ty: &'ll Type, | |
1472 | vals: &[&'ll Value], | |
1473 | bbs: &[&'ll BasicBlock], | |
1474 | ) -> &'ll Value { | |
532ac7d7 | 1475 | assert_eq!(vals.len(), bbs.len()); |
dfeec247 | 1476 | let phi = unsafe { llvm::LLVMBuildPhi(self.llbuilder, ty, UNNAMED) }; |
532ac7d7 | 1477 | unsafe { |
dfeec247 | 1478 | llvm::LLVMAddIncoming(phi, vals.as_ptr(), bbs.as_ptr(), vals.len() as c_uint); |
532ac7d7 XL |
1479 | phi |
1480 | } | |
1481 | } | |
1482 | ||
1483 | fn add_incoming_to_phi(&mut self, phi: &'ll Value, val: &'ll Value, bb: &'ll BasicBlock) { | |
532ac7d7 XL |
1484 | unsafe { |
1485 | llvm::LLVMAddIncoming(phi, &val, &bb, 1 as c_uint); | |
1486 | } | |
1487 | } | |
3dfed10e | 1488 | |
f2b60f7d FG |
1489 | fn fptoint_sat(&mut self, signed: bool, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { |
1490 | let src_ty = self.cx.val_ty(val); | |
1491 | let (float_ty, int_ty, vector_length) = if self.cx.type_kind(src_ty) == TypeKind::Vector { | |
1492 | assert_eq!(self.cx.vector_length(src_ty), self.cx.vector_length(dest_ty)); | |
1493 | ( | |
1494 | self.cx.element_type(src_ty), | |
1495 | self.cx.element_type(dest_ty), | |
1496 | Some(self.cx.vector_length(src_ty)), | |
1497 | ) | |
5099ac24 | 1498 | } else { |
f2b60f7d FG |
1499 | (src_ty, dest_ty, None) |
1500 | }; | |
1501 | let float_width = self.cx.float_width(float_ty); | |
1502 | let int_width = self.cx.int_width(int_ty); | |
1503 | ||
1504 | let instr = if signed { "fptosi" } else { "fptoui" }; | |
1505 | let name = if let Some(vector_length) = vector_length { | |
1506 | format!( | |
1507 | "llvm.{}.sat.v{}i{}.v{}f{}", | |
1508 | instr, vector_length, int_width, vector_length, float_width | |
1509 | ) | |
1510 | } else { | |
1511 | format!("llvm.{}.sat.i{}.f{}", instr, int_width, float_width) | |
1512 | }; | |
1513 | let f = self.declare_cfn(&name, llvm::UnnamedAddr::No, self.type_func(&[src_ty], dest_ty)); | |
2b03887a | 1514 | self.call(self.type_func(&[src_ty], dest_ty), None, f, &[val], None) |
5099ac24 FG |
1515 | } |
1516 | ||
1517 | pub(crate) fn landing_pad( | |
1518 | &mut self, | |
1519 | ty: &'ll Type, | |
1520 | pers_fn: &'ll Value, | |
1521 | num_clauses: usize, | |
1522 | ) -> &'ll Value { | |
1523 | // Use LLVMSetPersonalityFn to set the personality. It supports arbitrary Consts while, | |
1524 | // LLVMBuildLandingPad requires the argument to be a Function (as of LLVM 12). The | |
1525 | // personality lives on the parent function anyway. | |
1526 | self.set_personality_fn(pers_fn); | |
1527 | unsafe { | |
1528 | llvm::LLVMBuildLandingPad(self.llbuilder, ty, None, num_clauses as c_uint, UNNAMED) | |
1529 | } | |
1530 | } | |
a7813a04 | 1531 | } |