]>
Commit | Line | Data |
---|---|---|
94222f64 | 1 | use crate::abi::{Abi, FnAbi, FnAbiLlvmExt, LlvmType, PassMode}; |
dfeec247 | 2 | use crate::builder::Builder; |
9fa01778 | 3 | use crate::context::CodegenCx; |
dfeec247 | 4 | use crate::llvm; |
9fa01778 XL |
5 | use crate::type_::Type; |
6 | use crate::type_of::LayoutLlvmExt; | |
9fa01778 | 7 | use crate::va_arg::emit_va_arg; |
dfeec247 | 8 | use crate::value::Value; |
ba9703b0 | 9 | |
fe692bf9 | 10 | use rustc_codegen_ssa::base::{compare_simd_types, wants_msvc_seh, wants_wasm_eh}; |
f035d41b | 11 | use rustc_codegen_ssa::common::{IntPredicate, TypeKind}; |
9c376795 | 12 | use rustc_codegen_ssa::errors::{ExpectedPointerMutability, InvalidMonomorphization}; |
1b1a35ee | 13 | use rustc_codegen_ssa::mir::operand::OperandRef; |
dfeec247 | 14 | use rustc_codegen_ssa::mir::place::PlaceRef; |
ba9703b0 | 15 | use rustc_codegen_ssa::traits::*; |
dfeec247 | 16 | use rustc_hir as hir; |
c295e0f8 | 17 | use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, LayoutOf}; |
781aab86 | 18 | use rustc_middle::ty::{self, GenericArgsRef, Ty}; |
ba9703b0 | 19 | use rustc_middle::{bug, span_bug}; |
3dfed10e | 20 | use rustc_span::{sym, symbol::kw, Span, Symbol}; |
3c0e092e | 21 | use rustc_target::abi::{self, Align, HasDataLayout, Primitive}; |
c295e0f8 | 22 | use rustc_target::spec::{HasTargetSpec, PanicStrategy}; |
32a655c1 | 23 | |
e9174d1e SL |
24 | use std::cmp::Ordering; |
25 | ||
a2a8927a XL |
26 | fn get_simple_intrinsic<'ll>( |
27 | cx: &CodegenCx<'ll, '_>, | |
28 | name: Symbol, | |
29 | ) -> Option<(&'ll Type, &'ll Value)> { | |
54a0048b | 30 | let llvm_name = match name { |
3dfed10e XL |
31 | sym::sqrtf32 => "llvm.sqrt.f32", |
32 | sym::sqrtf64 => "llvm.sqrt.f64", | |
33 | sym::powif32 => "llvm.powi.f32", | |
34 | sym::powif64 => "llvm.powi.f64", | |
35 | sym::sinf32 => "llvm.sin.f32", | |
36 | sym::sinf64 => "llvm.sin.f64", | |
37 | sym::cosf32 => "llvm.cos.f32", | |
38 | sym::cosf64 => "llvm.cos.f64", | |
39 | sym::powf32 => "llvm.pow.f32", | |
40 | sym::powf64 => "llvm.pow.f64", | |
41 | sym::expf32 => "llvm.exp.f32", | |
42 | sym::expf64 => "llvm.exp.f64", | |
43 | sym::exp2f32 => "llvm.exp2.f32", | |
44 | sym::exp2f64 => "llvm.exp2.f64", | |
45 | sym::logf32 => "llvm.log.f32", | |
46 | sym::logf64 => "llvm.log.f64", | |
47 | sym::log10f32 => "llvm.log10.f32", | |
48 | sym::log10f64 => "llvm.log10.f64", | |
49 | sym::log2f32 => "llvm.log2.f32", | |
50 | sym::log2f64 => "llvm.log2.f64", | |
51 | sym::fmaf32 => "llvm.fma.f32", | |
52 | sym::fmaf64 => "llvm.fma.f64", | |
53 | sym::fabsf32 => "llvm.fabs.f32", | |
54 | sym::fabsf64 => "llvm.fabs.f64", | |
55 | sym::minnumf32 => "llvm.minnum.f32", | |
56 | sym::minnumf64 => "llvm.minnum.f64", | |
57 | sym::maxnumf32 => "llvm.maxnum.f32", | |
58 | sym::maxnumf64 => "llvm.maxnum.f64", | |
59 | sym::copysignf32 => "llvm.copysign.f32", | |
60 | sym::copysignf64 => "llvm.copysign.f64", | |
61 | sym::floorf32 => "llvm.floor.f32", | |
62 | sym::floorf64 => "llvm.floor.f64", | |
63 | sym::ceilf32 => "llvm.ceil.f32", | |
64 | sym::ceilf64 => "llvm.ceil.f64", | |
65 | sym::truncf32 => "llvm.trunc.f32", | |
66 | sym::truncf64 => "llvm.trunc.f64", | |
67 | sym::rintf32 => "llvm.rint.f32", | |
68 | sym::rintf64 => "llvm.rint.f64", | |
69 | sym::nearbyintf32 => "llvm.nearbyint.f32", | |
70 | sym::nearbyintf64 => "llvm.nearbyint.f64", | |
71 | sym::roundf32 => "llvm.round.f32", | |
72 | sym::roundf64 => "llvm.round.f64", | |
f2b60f7d | 73 | sym::ptr_mask => "llvm.ptrmask", |
353b0b11 FG |
74 | sym::roundevenf32 => "llvm.roundeven.f32", |
75 | sym::roundevenf64 => "llvm.roundeven.f64", | |
dfeec247 | 76 | _ => return None, |
1a4d82fc | 77 | }; |
c295e0f8 | 78 | Some(cx.get_intrinsic(llvm_name)) |
1a4d82fc JJ |
79 | } |
80 | ||
a2a8927a | 81 | impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { |
a1dfa0c6 XL |
82 | fn codegen_intrinsic_call( |
83 | &mut self, | |
e1599b0c | 84 | instance: ty::Instance<'tcx>, |
60c5eb7d | 85 | fn_abi: &FnAbi<'tcx, Ty<'tcx>>, |
a1dfa0c6 XL |
86 | args: &[OperandRef<'tcx, &'ll Value>], |
87 | llresult: &'ll Value, | |
88 | span: Span, | |
89 | ) { | |
90 | let tcx = self.tcx; | |
3dfed10e | 91 | let callee_ty = instance.ty(tcx, ty::ParamEnv::reveal_all()); |
c34b1796 | 92 | |
add651ee | 93 | let ty::FnDef(def_id, fn_args) = *callee_ty.kind() else { |
5e7ed085 | 94 | bug!("expected fn item type, found {}", callee_ty); |
a1dfa0c6 | 95 | }; |
9e0c209e | 96 | |
a1dfa0c6 | 97 | let sig = callee_ty.fn_sig(tcx); |
fc512014 | 98 | let sig = tcx.normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), sig); |
a1dfa0c6 XL |
99 | let arg_tys = sig.inputs(); |
100 | let ret_ty = sig.output(); | |
3dfed10e | 101 | let name = tcx.item_name(def_id); |
a1dfa0c6 XL |
102 | |
103 | let llret_ty = self.layout_of(ret_ty).llvm_type(self); | |
60c5eb7d | 104 | let result = PlaceRef::new_sized(llresult, fn_abi.ret.layout); |
a1dfa0c6 XL |
105 | |
106 | let simple = get_simple_intrinsic(self, name); | |
107 | let llval = match name { | |
94222f64 XL |
108 | _ if simple.is_some() => { |
109 | let (simple_ty, simple_fn) = simple.unwrap(); | |
110 | self.call( | |
111 | simple_ty, | |
2b03887a | 112 | None, |
49aad941 | 113 | None, |
94222f64 XL |
114 | simple_fn, |
115 | &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(), | |
116 | None, | |
117 | ) | |
d9579d0f | 118 | } |
94222f64 XL |
119 | sym::likely => { |
120 | self.call_intrinsic("llvm.expect.i1", &[args[0].immediate(), self.const_bool(true)]) | |
1a4d82fc | 121 | } |
94222f64 XL |
122 | sym::unlikely => self |
123 | .call_intrinsic("llvm.expect.i1", &[args[0].immediate(), self.const_bool(false)]), | |
3dfed10e | 124 | kw::Try => { |
dfeec247 XL |
125 | try_intrinsic( |
126 | self, | |
127 | args[0].immediate(), | |
128 | args[1].immediate(), | |
129 | args[2].immediate(), | |
130 | llresult, | |
131 | ); | |
a1dfa0c6 | 132 | return; |
54a0048b | 133 | } |
94222f64 | 134 | sym::breakpoint => self.call_intrinsic("llvm.debugtrap", &[]), |
3dfed10e | 135 | sym::va_copy => { |
94222f64 | 136 | self.call_intrinsic("llvm.va_copy", &[args[0].immediate(), args[1].immediate()]) |
a1dfa0c6 | 137 | } |
3dfed10e | 138 | sym::va_arg => { |
60c5eb7d | 139 | match fn_abi.ret.layout.abi { |
c295e0f8 | 140 | abi::Abi::Scalar(scalar) => { |
04454e1e | 141 | match scalar.primitive() { |
a1dfa0c6 XL |
142 | Primitive::Int(..) => { |
143 | if self.cx().size_of(ret_ty).bytes() < 4 { | |
94222f64 | 144 | // `va_arg` should not be called on an integer type |
a1dfa0c6 | 145 | // less than 4 bytes in length. If it is, promote |
94222f64 | 146 | // the integer to an `i32` and truncate the result |
a1dfa0c6 | 147 | // back to the smaller type. |
dfeec247 | 148 | let promoted_result = emit_va_arg(self, args[0], tcx.types.i32); |
a1dfa0c6 XL |
149 | self.trunc(promoted_result, llret_ty) |
150 | } else { | |
151 | emit_va_arg(self, args[0], ret_ty) | |
152 | } | |
153 | } | |
9ffffee4 | 154 | Primitive::F64 | Primitive::Pointer(_) => { |
a1dfa0c6 XL |
155 | emit_va_arg(self, args[0], ret_ty) |
156 | } | |
157 | // `va_arg` should never be used with the return type f32. | |
dfeec247 | 158 | Primitive::F32 => bug!("the va_arg intrinsic does not work with `f32`"), |
92a42be0 | 159 | } |
a1dfa0c6 | 160 | } |
dfeec247 | 161 | _ => bug!("the va_arg intrinsic does not work with non-scalar types"), |
54a0048b SL |
162 | } |
163 | } | |
32a655c1 | 164 | |
3dfed10e | 165 | sym::volatile_load | sym::unaligned_volatile_load => { |
add651ee | 166 | let tp_ty = fn_args.type_at(0); |
136023e0 | 167 | let ptr = args[0].immediate(); |
781aab86 | 168 | let load = if let PassMode::Cast { cast: ty, pad_i32: _ } = &fn_abi.ret.mode { |
136023e0 | 169 | let llty = ty.llvm_type(self); |
136023e0 XL |
170 | self.volatile_load(llty, ptr) |
171 | } else { | |
172 | self.volatile_load(self.layout_of(tp_ty).llvm_type(self), ptr) | |
173 | }; | |
3dfed10e | 174 | let align = if name == sym::unaligned_volatile_load { |
a1dfa0c6 XL |
175 | 1 |
176 | } else { | |
177 | self.align_of(tp_ty).bytes() as u32 | |
178 | }; | |
179 | unsafe { | |
180 | llvm::LLVMSetAlignment(load, align); | |
181 | } | |
1b1a35ee | 182 | self.to_immediate(load, self.layout_of(tp_ty)) |
dfeec247 | 183 | } |
3dfed10e | 184 | sym::volatile_store => { |
a1dfa0c6 XL |
185 | let dst = args[0].deref(self.cx()); |
186 | args[1].val.volatile_store(self, dst); | |
187 | return; | |
dfeec247 | 188 | } |
3dfed10e | 189 | sym::unaligned_volatile_store => { |
a1dfa0c6 XL |
190 | let dst = args[0].deref(self.cx()); |
191 | args[1].val.unaligned_volatile_store(self, dst); | |
192 | return; | |
dfeec247 | 193 | } |
3dfed10e XL |
194 | sym::prefetch_read_data |
195 | | sym::prefetch_write_data | |
196 | | sym::prefetch_read_instruction | |
197 | | sym::prefetch_write_instruction => { | |
a1dfa0c6 | 198 | let (rw, cache_type) = match name { |
3dfed10e XL |
199 | sym::prefetch_read_data => (0, 1), |
200 | sym::prefetch_write_data => (1, 1), | |
201 | sym::prefetch_read_instruction => (0, 0), | |
202 | sym::prefetch_write_instruction => (1, 0), | |
dfeec247 | 203 | _ => bug!(), |
a1dfa0c6 | 204 | }; |
94222f64 XL |
205 | self.call_intrinsic( |
206 | "llvm.prefetch", | |
dfeec247 XL |
207 | &[ |
208 | args[0].immediate(), | |
209 | self.const_i32(rw), | |
210 | args[1].immediate(), | |
211 | self.const_i32(cache_type), | |
212 | ], | |
dfeec247 XL |
213 | ) |
214 | } | |
3dfed10e XL |
215 | sym::ctlz |
216 | | sym::ctlz_nonzero | |
217 | | sym::cttz | |
218 | | sym::cttz_nonzero | |
219 | | sym::ctpop | |
220 | | sym::bswap | |
221 | | sym::bitreverse | |
3dfed10e XL |
222 | | sym::rotate_left |
223 | | sym::rotate_right | |
224 | | sym::saturating_add | |
225 | | sym::saturating_sub => { | |
a1dfa0c6 XL |
226 | let ty = arg_tys[0]; |
227 | match int_type_width_signed(ty, self) { | |
dfeec247 | 228 | Some((width, signed)) => match name { |
3dfed10e | 229 | sym::ctlz | sym::cttz => { |
dfeec247 | 230 | let y = self.const_bool(false); |
94222f64 | 231 | self.call_intrinsic( |
add651ee | 232 | &format!("llvm.{name}.i{width}"), |
94222f64 XL |
233 | &[args[0].immediate(), y], |
234 | ) | |
dfeec247 | 235 | } |
c295e0f8 | 236 | sym::ctlz_nonzero => { |
dfeec247 | 237 | let y = self.const_bool(true); |
add651ee | 238 | let llvm_name = &format!("llvm.ctlz.i{width}"); |
c295e0f8 XL |
239 | self.call_intrinsic(llvm_name, &[args[0].immediate(), y]) |
240 | } | |
241 | sym::cttz_nonzero => { | |
242 | let y = self.const_bool(true); | |
add651ee | 243 | let llvm_name = &format!("llvm.cttz.i{width}"); |
94222f64 | 244 | self.call_intrinsic(llvm_name, &[args[0].immediate(), y]) |
dfeec247 | 245 | } |
94222f64 | 246 | sym::ctpop => self.call_intrinsic( |
add651ee | 247 | &format!("llvm.ctpop.i{width}"), |
dfeec247 | 248 | &[args[0].immediate()], |
dfeec247 | 249 | ), |
3dfed10e | 250 | sym::bswap => { |
dfeec247 XL |
251 | if width == 8 { |
252 | args[0].immediate() // byte swap a u8/i8 is just a no-op | |
253 | } else { | |
94222f64 | 254 | self.call_intrinsic( |
add651ee | 255 | &format!("llvm.bswap.i{width}"), |
a1dfa0c6 | 256 | &[args[0].immediate()], |
a1dfa0c6 XL |
257 | ) |
258 | } | |
dfeec247 | 259 | } |
94222f64 | 260 | sym::bitreverse => self.call_intrinsic( |
add651ee | 261 | &format!("llvm.bitreverse.i{width}"), |
dfeec247 | 262 | &[args[0].immediate()], |
dfeec247 | 263 | ), |
3dfed10e XL |
264 | sym::rotate_left | sym::rotate_right => { |
265 | let is_left = name == sym::rotate_left; | |
dfeec247 XL |
266 | let val = args[0].immediate(); |
267 | let raw_shift = args[1].immediate(); | |
268 | // rotate = funnel shift with first two args the same | |
269 | let llvm_name = | |
270 | &format!("llvm.fsh{}.i{}", if is_left { 'l' } else { 'r' }, width); | |
94222f64 | 271 | self.call_intrinsic(llvm_name, &[val, val, raw_shift]) |
dfeec247 | 272 | } |
3dfed10e XL |
273 | sym::saturating_add | sym::saturating_sub => { |
274 | let is_add = name == sym::saturating_add; | |
dfeec247 XL |
275 | let lhs = args[0].immediate(); |
276 | let rhs = args[1].immediate(); | |
ba9703b0 XL |
277 | let llvm_name = &format!( |
278 | "llvm.{}{}.sat.i{}", | |
279 | if signed { 's' } else { 'u' }, | |
280 | if is_add { "add" } else { "sub" }, | |
281 | width | |
282 | ); | |
94222f64 | 283 | self.call_intrinsic(llvm_name, &[lhs, rhs]) |
dfeec247 XL |
284 | } |
285 | _ => bug!(), | |
286 | }, | |
a1dfa0c6 | 287 | None => { |
9c376795 | 288 | tcx.sess.emit_err(InvalidMonomorphization::BasicIntegerType { |
dfeec247 | 289 | span, |
9c376795 FG |
290 | name, |
291 | ty, | |
292 | }); | |
a1dfa0c6 | 293 | return; |
54a0048b | 294 | } |
1a4d82fc | 295 | } |
dfeec247 | 296 | } |
1a4d82fc | 297 | |
136023e0 XL |
298 | sym::raw_eq => { |
299 | use abi::Abi::*; | |
add651ee | 300 | let tp_ty = fn_args.type_at(0); |
136023e0 | 301 | let layout = self.layout_of(tp_ty).layout; |
5e7ed085 | 302 | let use_integer_compare = match layout.abi() { |
136023e0 XL |
303 | Scalar(_) | ScalarPair(_, _) => true, |
304 | Uninhabited | Vector { .. } => false, | |
305 | Aggregate { .. } => { | |
306 | // For rusty ABIs, small aggregates are actually passed | |
307 | // as `RegKind::Integer` (see `FnAbi::adjust_for_abi`), | |
308 | // so we re-use that same threshold here. | |
5e7ed085 | 309 | layout.size() <= self.data_layout().pointer_size * 2 |
136023e0 XL |
310 | } |
311 | }; | |
312 | ||
313 | let a = args[0].immediate(); | |
314 | let b = args[1].immediate(); | |
5e7ed085 | 315 | if layout.size().bytes() == 0 { |
136023e0 XL |
316 | self.const_bool(true) |
317 | } else if use_integer_compare { | |
5e7ed085 | 318 | let integer_ty = self.type_ix(layout.size().bits()); |
add651ee FG |
319 | let a_val = self.load(integer_ty, a, layout.align().abi); |
320 | let b_val = self.load(integer_ty, b, layout.align().abi); | |
136023e0 XL |
321 | self.icmp(IntPredicate::IntEQ, a_val, b_val) |
322 | } else { | |
5e7ed085 | 323 | let n = self.const_usize(layout.size().bytes()); |
add651ee | 324 | let cmp = self.call_intrinsic("memcmp", &[a, b, n]); |
5e7ed085 FG |
325 | match self.cx.sess().target.arch.as_ref() { |
326 | "avr" | "msp430" => self.icmp(IntPredicate::IntEQ, cmp, self.const_i16(0)), | |
327 | _ => self.icmp(IntPredicate::IntEQ, cmp, self.const_i32(0)), | |
328 | } | |
136023e0 XL |
329 | } |
330 | } | |
331 | ||
add651ee FG |
332 | sym::compare_bytes => { |
333 | // Here we assume that the `memcmp` provided by the target is a NOP for size 0. | |
334 | let cmp = self.call_intrinsic( | |
335 | "memcmp", | |
336 | &[args[0].immediate(), args[1].immediate(), args[2].immediate()], | |
337 | ); | |
338 | // Some targets have `memcmp` returning `i16`, but the intrinsic is always `i32`. | |
339 | self.sext(cmp, self.type_ix(32)) | |
340 | } | |
341 | ||
94222f64 XL |
342 | sym::black_box => { |
343 | args[0].val.store(self, result); | |
487cf647 | 344 | let result_val_span = [result.llval]; |
94222f64 XL |
345 | // We need to "use" the argument in some way LLVM can't introspect, and on |
346 | // targets that support it we can typically leverage inline assembly to do | |
347 | // this. LLVM's interpretation of inline assembly is that it's, well, a black | |
348 | // box. This isn't the greatest implementation since it probably deoptimizes | |
349 | // more than we want, but it's so far good enough. | |
487cf647 FG |
350 | // |
351 | // For zero-sized types, the location pointed to by the result may be | |
352 | // uninitialized. Do not "use" the result in this case; instead just clobber | |
353 | // the memory. | |
354 | let (constraint, inputs): (&str, &[_]) = if result.layout.is_zst() { | |
355 | ("~{memory}", &[]) | |
356 | } else { | |
357 | ("r,~{memory}", &result_val_span) | |
358 | }; | |
94222f64 XL |
359 | crate::asm::inline_asm_call( |
360 | self, | |
361 | "", | |
487cf647 FG |
362 | constraint, |
363 | inputs, | |
94222f64 XL |
364 | self.type_void(), |
365 | true, | |
366 | false, | |
5099ac24 | 367 | llvm::AsmDialect::Att, |
94222f64 | 368 | &[span], |
a2a8927a XL |
369 | false, |
370 | None, | |
94222f64 XL |
371 | ) |
372 | .unwrap_or_else(|| bug!("failed to generate inline asm call for `black_box`")); | |
373 | ||
374 | // We have copied the value to `result` already. | |
375 | return; | |
376 | } | |
377 | ||
c295e0f8 | 378 | _ if name.as_str().starts_with("simd_") => { |
781aab86 FG |
379 | match generic_simd_intrinsic( |
380 | self, name, callee_ty, fn_args, args, ret_ty, llret_ty, span, | |
381 | ) { | |
a1dfa0c6 | 382 | Ok(llval) => llval, |
dfeec247 | 383 | Err(()) => return, |
1a4d82fc JJ |
384 | } |
385 | } | |
e74abb32 | 386 | |
353b0b11 | 387 | _ => bug!("unknown intrinsic '{}' -- should it have been lowered earlier?", name), |
a1dfa0c6 | 388 | }; |
1a4d82fc | 389 | |
60c5eb7d | 390 | if !fn_abi.ret.is_ignore() { |
781aab86 | 391 | if let PassMode::Cast { .. } = &fn_abi.ret.mode { |
add651ee | 392 | self.store(llval, result.llval, result.align); |
a1dfa0c6 XL |
393 | } else { |
394 | OperandRef::from_immediate_or_packed_pair(self, llval, result.layout) | |
dfeec247 XL |
395 | .val |
396 | .store(self, result); | |
a1dfa0c6 | 397 | } |
54a0048b | 398 | } |
1a4d82fc | 399 | } |
a1dfa0c6 XL |
400 | |
401 | fn abort(&mut self) { | |
94222f64 | 402 | self.call_intrinsic("llvm.trap", &[]); |
a1dfa0c6 XL |
403 | } |
404 | ||
405 | fn assume(&mut self, val: Self::Value) { | |
94222f64 | 406 | self.call_intrinsic("llvm.assume", &[val]); |
a1dfa0c6 XL |
407 | } |
408 | ||
409 | fn expect(&mut self, cond: Self::Value, expected: bool) -> Self::Value { | |
94222f64 | 410 | self.call_intrinsic("llvm.expect.i1", &[cond, self.const_bool(expected)]) |
a1dfa0c6 | 411 | } |
532ac7d7 | 412 | |
3c0e092e XL |
413 | fn type_test(&mut self, pointer: Self::Value, typeid: Self::Value) -> Self::Value { |
414 | // Test the called operand using llvm.type.test intrinsic. The LowerTypeTests link-time | |
415 | // optimization pass replaces calls to this intrinsic with code to test type membership. | |
add651ee | 416 | self.call_intrinsic("llvm.type.test", &[pointer, typeid]) |
e74abb32 XL |
417 | } |
418 | ||
923072b8 FG |
419 | fn type_checked_load( |
420 | &mut self, | |
421 | llvtable: &'ll Value, | |
422 | vtable_byte_offset: u64, | |
423 | typeid: &'ll Value, | |
424 | ) -> Self::Value { | |
425 | let vtable_byte_offset = self.const_i32(vtable_byte_offset as i32); | |
9c376795 FG |
426 | let type_checked_load = |
427 | self.call_intrinsic("llvm.type.checked.load", &[llvtable, vtable_byte_offset, typeid]); | |
428 | self.extract_value(type_checked_load, 0) | |
923072b8 FG |
429 | } |
430 | ||
dc9dc135 | 431 | fn va_start(&mut self, va_list: &'ll Value) -> &'ll Value { |
94222f64 | 432 | self.call_intrinsic("llvm.va_start", &[va_list]) |
532ac7d7 XL |
433 | } |
434 | ||
dc9dc135 | 435 | fn va_end(&mut self, va_list: &'ll Value) -> &'ll Value { |
94222f64 | 436 | self.call_intrinsic("llvm.va_end", &[va_list]) |
532ac7d7 | 437 | } |
1a4d82fc JJ |
438 | } |
439 | ||
a2a8927a XL |
440 | fn try_intrinsic<'ll>( |
441 | bx: &mut Builder<'_, 'll, '_>, | |
ba9703b0 | 442 | try_func: &'ll Value, |
b7449926 | 443 | data: &'ll Value, |
ba9703b0 | 444 | catch_func: &'ll Value, |
b7449926 | 445 | dest: &'ll Value, |
32a655c1 | 446 | ) { |
f9f354fc | 447 | if bx.sess().panic_strategy() == PanicStrategy::Abort { |
add651ee | 448 | let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void()); |
49aad941 | 449 | bx.call(try_func_ty, None, None, try_func, &[data], None); |
ba9703b0 XL |
450 | // Return 0 unconditionally from the intrinsic call; |
451 | // we can never unwind. | |
452 | let ret_align = bx.tcx().data_layout.i32_align.abi; | |
453 | bx.store(bx.const_i32(0), dest, ret_align); | |
2c00a5a8 | 454 | } else if wants_msvc_seh(bx.sess()) { |
ba9703b0 | 455 | codegen_msvc_try(bx, try_func, data, catch_func, dest); |
fe692bf9 FG |
456 | } else if wants_wasm_eh(bx.sess()) { |
457 | codegen_wasm_try(bx, try_func, data, catch_func, dest); | |
923072b8 | 458 | } else if bx.sess().target.os == "emscripten" { |
1b1a35ee | 459 | codegen_emcc_try(bx, try_func, data, catch_func, dest); |
c1a9b12d | 460 | } else { |
ba9703b0 | 461 | codegen_gnu_try(bx, try_func, data, catch_func, dest); |
c1a9b12d SL |
462 | } |
463 | } | |
464 | ||
7453a54e | 465 | // MSVC's definition of the `rust_try` function. |
c1a9b12d | 466 | // |
7453a54e SL |
467 | // This implementation uses the new exception handling instructions in LLVM |
468 | // which have support in LLVM for SEH on MSVC targets. Although these | |
469 | // instructions are meant to work for all targets, as of the time of this | |
470 | // writing, however, LLVM does not recommend the usage of these new instructions | |
471 | // as the old ones are still more optimized. | |
a2a8927a XL |
472 | fn codegen_msvc_try<'ll>( |
473 | bx: &mut Builder<'_, 'll, '_>, | |
ba9703b0 | 474 | try_func: &'ll Value, |
b7449926 | 475 | data: &'ll Value, |
ba9703b0 | 476 | catch_func: &'ll Value, |
b7449926 XL |
477 | dest: &'ll Value, |
478 | ) { | |
94222f64 | 479 | let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| { |
a1dfa0c6 | 480 | bx.set_personality_fn(bx.eh_personality()); |
32a655c1 | 481 | |
5e7ed085 FG |
482 | let normal = bx.append_sibling_block("normal"); |
483 | let catchswitch = bx.append_sibling_block("catchswitch"); | |
484 | let catchpad_rust = bx.append_sibling_block("catchpad_rust"); | |
485 | let catchpad_foreign = bx.append_sibling_block("catchpad_foreign"); | |
486 | let caught = bx.append_sibling_block("caught"); | |
32a655c1 | 487 | |
ba9703b0 | 488 | let try_func = llvm::get_param(bx.llfn(), 0); |
2c00a5a8 | 489 | let data = llvm::get_param(bx.llfn(), 1); |
ba9703b0 | 490 | let catch_func = llvm::get_param(bx.llfn(), 2); |
c1a9b12d | 491 | |
7453a54e SL |
492 | // We're generating an IR snippet that looks like: |
493 | // | |
ba9703b0 | 494 | // declare i32 @rust_try(%try_func, %data, %catch_func) { |
1b1a35ee | 495 | // %slot = alloca i8* |
ba9703b0 | 496 | // invoke %try_func(%data) to label %normal unwind label %catchswitch |
7453a54e SL |
497 | // |
498 | // normal: | |
499 | // ret i32 0 | |
c1a9b12d | 500 | // |
7453a54e | 501 | // catchswitch: |
1b1a35ee | 502 | // %cs = catchswitch within none [%catchpad_rust, %catchpad_foreign] unwind to caller |
c1a9b12d | 503 | // |
1b1a35ee XL |
504 | // catchpad_rust: |
505 | // %tok = catchpad within %cs [%type_descriptor, 8, %slot] | |
ba9703b0 XL |
506 | // %ptr = load %slot |
507 | // call %catch_func(%data, %ptr) | |
7453a54e | 508 | // catchret from %tok to label %caught |
c1a9b12d | 509 | // |
1b1a35ee XL |
510 | // catchpad_foreign: |
511 | // %tok = catchpad within %cs [null, 64, null] | |
512 | // call %catch_func(%data, null) | |
513 | // catchret from %tok to label %caught | |
514 | // | |
7453a54e SL |
515 | // caught: |
516 | // ret i32 1 | |
517 | // } | |
c1a9b12d | 518 | // |
a7813a04 XL |
519 | // This structure follows the basic usage of throw/try/catch in LLVM. |
520 | // For example, compile this C++ snippet to see what LLVM generates: | |
521 | // | |
e74abb32 | 522 | // struct rust_panic { |
dfeec247 XL |
523 | // rust_panic(const rust_panic&); |
524 | // ~rust_panic(); | |
525 | // | |
1b1a35ee | 526 | // void* x[2]; |
ba9703b0 | 527 | // }; |
e74abb32 | 528 | // |
ba9703b0 XL |
529 | // int __rust_try( |
530 | // void (*try_func)(void*), | |
531 | // void *data, | |
532 | // void (*catch_func)(void*, void*) noexcept | |
533 | // ) { | |
a7813a04 | 534 | // try { |
ba9703b0 | 535 | // try_func(data); |
a7813a04 | 536 | // return 0; |
dfeec247 | 537 | // } catch(rust_panic& a) { |
ba9703b0 | 538 | // catch_func(data, &a); |
a7813a04 | 539 | // return 1; |
1b1a35ee XL |
540 | // } catch(...) { |
541 | // catch_func(data, NULL); | |
542 | // return 1; | |
a7813a04 XL |
543 | // } |
544 | // } | |
7453a54e SL |
545 | // |
546 | // More information can be found in libstd's seh.rs implementation. | |
dfeec247 | 547 | let ptr_align = bx.tcx().data_layout.pointer_align.abi; |
add651ee FG |
548 | let slot = bx.alloca(bx.type_ptr(), ptr_align); |
549 | let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void()); | |
49aad941 | 550 | bx.invoke(try_func_ty, None, None, try_func, &[data], normal, catchswitch, None); |
7453a54e | 551 | |
5e7ed085 FG |
552 | bx.switch_to_block(normal); |
553 | bx.ret(bx.const_i32(0)); | |
7453a54e | 554 | |
5e7ed085 FG |
555 | bx.switch_to_block(catchswitch); |
556 | let cs = bx.catch_switch(None, None, &[catchpad_rust, catchpad_foreign]); | |
7453a54e | 557 | |
ba9703b0 XL |
558 | // We can't use the TypeDescriptor defined in libpanic_unwind because it |
559 | // might be in another DLL and the SEH encoding only supports specifying | |
560 | // a TypeDescriptor from the current module. | |
561 | // | |
562 | // However this isn't an issue since the MSVC runtime uses string | |
563 | // comparison on the type name to match TypeDescriptors rather than | |
564 | // pointer equality. | |
565 | // | |
566 | // So instead we generate a new TypeDescriptor in each module that uses | |
567 | // `try` and let the linker merge duplicate definitions in the same | |
568 | // module. | |
569 | // | |
570 | // When modifying, make sure that the type_name string exactly matches | |
9c376795 | 571 | // the one used in library/panic_unwind/src/seh.rs. |
add651ee | 572 | let type_info_vtable = bx.declare_global("??_7type_info@@6B@", bx.type_ptr()); |
ba9703b0 XL |
573 | let type_name = bx.const_bytes(b"rust_panic\0"); |
574 | let type_info = | |
add651ee | 575 | bx.const_struct(&[type_info_vtable, bx.const_null(bx.type_ptr()), type_name], false); |
ba9703b0 XL |
576 | let tydesc = bx.declare_global("__rust_panic_type_info", bx.val_ty(type_info)); |
577 | unsafe { | |
578 | llvm::LLVMRustSetLinkage(tydesc, llvm::Linkage::LinkOnceODRLinkage); | |
579 | llvm::SetUniqueComdat(bx.llmod, tydesc); | |
580 | llvm::LLVMSetInitializer(tydesc, type_info); | |
581 | } | |
582 | ||
dfeec247 XL |
583 | // The flag value of 8 indicates that we are catching the exception by |
584 | // reference instead of by value. We can't use catch by value because | |
585 | // that requires copying the exception object, which we don't support | |
586 | // since our exception object effectively contains a Box. | |
587 | // | |
588 | // Source: MicrosoftCXXABI::getAddrOfCXXCatchHandlerType in clang | |
5e7ed085 | 589 | bx.switch_to_block(catchpad_rust); |
dfeec247 | 590 | let flags = bx.const_i32(8); |
5e7ed085 | 591 | let funclet = bx.catch_pad(cs, &[tydesc, flags, slot]); |
add651ee FG |
592 | let ptr = bx.load(bx.type_ptr(), slot, ptr_align); |
593 | let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void()); | |
49aad941 | 594 | bx.call(catch_ty, None, None, catch_func, &[data, ptr], Some(&funclet)); |
5e7ed085 | 595 | bx.catch_ret(&funclet, caught); |
1b1a35ee XL |
596 | |
597 | // The flag value of 64 indicates a "catch-all". | |
5e7ed085 | 598 | bx.switch_to_block(catchpad_foreign); |
1b1a35ee | 599 | let flags = bx.const_i32(64); |
add651ee | 600 | let null = bx.const_null(bx.type_ptr()); |
5e7ed085 | 601 | let funclet = bx.catch_pad(cs, &[null, flags, null]); |
49aad941 | 602 | bx.call(catch_ty, None, None, catch_func, &[data, null], Some(&funclet)); |
5e7ed085 | 603 | bx.catch_ret(&funclet, caught); |
7453a54e | 604 | |
5e7ed085 FG |
605 | bx.switch_to_block(caught); |
606 | bx.ret(bx.const_i32(1)); | |
c1a9b12d SL |
607 | }); |
608 | ||
609 | // Note that no invoke is used here because by definition this function | |
610 | // can't panic (that's what it's catching). | |
49aad941 | 611 | let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None); |
a1dfa0c6 | 612 | let i32_align = bx.tcx().data_layout.i32_align.abi; |
2c00a5a8 | 613 | bx.store(ret, dest, i32_align); |
c1a9b12d SL |
614 | } |
615 | ||
fe692bf9 FG |
616 | // WASM's definition of the `rust_try` function. |
617 | fn codegen_wasm_try<'ll>( | |
618 | bx: &mut Builder<'_, 'll, '_>, | |
619 | try_func: &'ll Value, | |
620 | data: &'ll Value, | |
621 | catch_func: &'ll Value, | |
622 | dest: &'ll Value, | |
623 | ) { | |
624 | let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| { | |
625 | bx.set_personality_fn(bx.eh_personality()); | |
626 | ||
627 | let normal = bx.append_sibling_block("normal"); | |
628 | let catchswitch = bx.append_sibling_block("catchswitch"); | |
629 | let catchpad = bx.append_sibling_block("catchpad"); | |
630 | let caught = bx.append_sibling_block("caught"); | |
631 | ||
632 | let try_func = llvm::get_param(bx.llfn(), 0); | |
633 | let data = llvm::get_param(bx.llfn(), 1); | |
634 | let catch_func = llvm::get_param(bx.llfn(), 2); | |
635 | ||
636 | // We're generating an IR snippet that looks like: | |
637 | // | |
638 | // declare i32 @rust_try(%try_func, %data, %catch_func) { | |
639 | // %slot = alloca i8* | |
640 | // invoke %try_func(%data) to label %normal unwind label %catchswitch | |
641 | // | |
642 | // normal: | |
643 | // ret i32 0 | |
644 | // | |
645 | // catchswitch: | |
646 | // %cs = catchswitch within none [%catchpad] unwind to caller | |
647 | // | |
648 | // catchpad: | |
649 | // %tok = catchpad within %cs [null] | |
650 | // %ptr = call @llvm.wasm.get.exception(token %tok) | |
651 | // %sel = call @llvm.wasm.get.ehselector(token %tok) | |
652 | // call %catch_func(%data, %ptr) | |
653 | // catchret from %tok to label %caught | |
654 | // | |
655 | // caught: | |
656 | // ret i32 1 | |
657 | // } | |
658 | // | |
add651ee | 659 | let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void()); |
fe692bf9 FG |
660 | bx.invoke(try_func_ty, None, None, try_func, &[data], normal, catchswitch, None); |
661 | ||
662 | bx.switch_to_block(normal); | |
663 | bx.ret(bx.const_i32(0)); | |
664 | ||
665 | bx.switch_to_block(catchswitch); | |
666 | let cs = bx.catch_switch(None, None, &[catchpad]); | |
667 | ||
668 | bx.switch_to_block(catchpad); | |
add651ee | 669 | let null = bx.const_null(bx.type_ptr()); |
fe692bf9 FG |
670 | let funclet = bx.catch_pad(cs, &[null]); |
671 | ||
672 | let ptr = bx.call_intrinsic("llvm.wasm.get.exception", &[funclet.cleanuppad()]); | |
673 | let _sel = bx.call_intrinsic("llvm.wasm.get.ehselector", &[funclet.cleanuppad()]); | |
674 | ||
add651ee | 675 | let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void()); |
fe692bf9 FG |
676 | bx.call(catch_ty, None, None, catch_func, &[data, ptr], Some(&funclet)); |
677 | bx.catch_ret(&funclet, caught); | |
678 | ||
679 | bx.switch_to_block(caught); | |
680 | bx.ret(bx.const_i32(1)); | |
681 | }); | |
682 | ||
683 | // Note that no invoke is used here because by definition this function | |
684 | // can't panic (that's what it's catching). | |
685 | let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None); | |
686 | let i32_align = bx.tcx().data_layout.i32_align.abi; | |
687 | bx.store(ret, dest, i32_align); | |
688 | } | |
689 | ||
dc9dc135 XL |
690 | // Definition of the standard `try` function for Rust using the GNU-like model |
691 | // of exceptions (e.g., the normal semantics of LLVM's `landingpad` and `invoke` | |
c1a9b12d SL |
692 | // instructions). |
693 | // | |
94b46f34 | 694 | // This codegen is a little surprising because we always call a shim |
7453a54e SL |
695 | // function instead of inlining the call to `invoke` manually here. This is done |
696 | // because in LLVM we're only allowed to have one personality per function | |
697 | // definition. The call to the `try` intrinsic is being inlined into the | |
698 | // function calling it, and that function may already have other personality | |
699 | // functions in play. By calling a shim we're guaranteed that our shim will have | |
700 | // the right personality function. | |
a2a8927a XL |
701 | fn codegen_gnu_try<'ll>( |
702 | bx: &mut Builder<'_, 'll, '_>, | |
ba9703b0 | 703 | try_func: &'ll Value, |
b7449926 | 704 | data: &'ll Value, |
ba9703b0 | 705 | catch_func: &'ll Value, |
b7449926 XL |
706 | dest: &'ll Value, |
707 | ) { | |
94222f64 | 708 | let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| { |
94b46f34 | 709 | // Codegens the shims described above: |
c1a9b12d | 710 | // |
2c00a5a8 | 711 | // bx: |
ba9703b0 | 712 | // invoke %try_func(%data) normal %normal unwind %catch |
c1a9b12d SL |
713 | // |
714 | // normal: | |
7453a54e | 715 | // ret 0 |
c1a9b12d SL |
716 | // |
717 | // catch: | |
ba9703b0 XL |
718 | // (%ptr, _) = landingpad |
719 | // call %catch_func(%data, %ptr) | |
7453a54e | 720 | // ret 1 |
5e7ed085 FG |
721 | let then = bx.append_sibling_block("then"); |
722 | let catch = bx.append_sibling_block("catch"); | |
c1a9b12d | 723 | |
ba9703b0 | 724 | let try_func = llvm::get_param(bx.llfn(), 0); |
2c00a5a8 | 725 | let data = llvm::get_param(bx.llfn(), 1); |
ba9703b0 | 726 | let catch_func = llvm::get_param(bx.llfn(), 2); |
add651ee | 727 | let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void()); |
49aad941 | 728 | bx.invoke(try_func_ty, None, None, try_func, &[data], then, catch, None); |
5e7ed085 FG |
729 | |
730 | bx.switch_to_block(then); | |
731 | bx.ret(bx.const_i32(0)); | |
c1a9b12d SL |
732 | |
733 | // Type indicator for the exception being thrown. | |
7453a54e SL |
734 | // |
735 | // The first value in this tuple is a pointer to the exception object | |
9c376795 | 736 | // being thrown. The second value is a "selector" indicating which of |
7453a54e SL |
737 | // the landing pad clauses the exception's type had been matched to. |
738 | // rust_try ignores the selector. | |
5e7ed085 | 739 | bx.switch_to_block(catch); |
add651ee | 740 | let lpad_ty = bx.type_struct(&[bx.type_ptr(), bx.type_i32()], false); |
5e7ed085 | 741 | let vals = bx.landing_pad(lpad_ty, bx.eh_personality(), 1); |
add651ee | 742 | let tydesc = bx.const_null(bx.type_ptr()); |
5e7ed085 FG |
743 | bx.add_clause(vals, tydesc); |
744 | let ptr = bx.extract_value(vals, 0); | |
add651ee | 745 | let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void()); |
49aad941 | 746 | bx.call(catch_ty, None, None, catch_func, &[data, ptr], None); |
5e7ed085 | 747 | bx.ret(bx.const_i32(1)); |
c1a9b12d SL |
748 | }); |
749 | ||
750 | // Note that no invoke is used here because by definition this function | |
751 | // can't panic (that's what it's catching). | |
49aad941 | 752 | let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None); |
a1dfa0c6 | 753 | let i32_align = bx.tcx().data_layout.i32_align.abi; |
2c00a5a8 | 754 | bx.store(ret, dest, i32_align); |
c1a9b12d SL |
755 | } |
756 | ||
1b1a35ee XL |
757 | // Variant of codegen_gnu_try used for emscripten where Rust panics are |
758 | // implemented using C++ exceptions. Here we use exceptions of a specific type | |
759 | // (`struct rust_panic`) to represent Rust panics. | |
a2a8927a XL |
760 | fn codegen_emcc_try<'ll>( |
761 | bx: &mut Builder<'_, 'll, '_>, | |
1b1a35ee XL |
762 | try_func: &'ll Value, |
763 | data: &'ll Value, | |
764 | catch_func: &'ll Value, | |
765 | dest: &'ll Value, | |
766 | ) { | |
94222f64 | 767 | let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| { |
1b1a35ee XL |
768 | // Codegens the shims described above: |
769 | // | |
770 | // bx: | |
771 | // invoke %try_func(%data) normal %normal unwind %catch | |
772 | // | |
773 | // normal: | |
774 | // ret 0 | |
775 | // | |
776 | // catch: | |
777 | // (%ptr, %selector) = landingpad | |
778 | // %rust_typeid = @llvm.eh.typeid.for(@_ZTI10rust_panic) | |
779 | // %is_rust_panic = %selector == %rust_typeid | |
780 | // %catch_data = alloca { i8*, i8 } | |
781 | // %catch_data[0] = %ptr | |
782 | // %catch_data[1] = %is_rust_panic | |
783 | // call %catch_func(%data, %catch_data) | |
784 | // ret 1 | |
5e7ed085 FG |
785 | let then = bx.append_sibling_block("then"); |
786 | let catch = bx.append_sibling_block("catch"); | |
1b1a35ee XL |
787 | |
788 | let try_func = llvm::get_param(bx.llfn(), 0); | |
789 | let data = llvm::get_param(bx.llfn(), 1); | |
790 | let catch_func = llvm::get_param(bx.llfn(), 2); | |
add651ee | 791 | let try_func_ty = bx.type_func(&[bx.type_ptr()], bx.type_void()); |
49aad941 | 792 | bx.invoke(try_func_ty, None, None, try_func, &[data], then, catch, None); |
5e7ed085 FG |
793 | |
794 | bx.switch_to_block(then); | |
795 | bx.ret(bx.const_i32(0)); | |
1b1a35ee XL |
796 | |
797 | // Type indicator for the exception being thrown. | |
798 | // | |
799 | // The first value in this tuple is a pointer to the exception object | |
9c376795 | 800 | // being thrown. The second value is a "selector" indicating which of |
1b1a35ee | 801 | // the landing pad clauses the exception's type had been matched to. |
5e7ed085 | 802 | bx.switch_to_block(catch); |
1b1a35ee | 803 | let tydesc = bx.eh_catch_typeinfo(); |
add651ee | 804 | let lpad_ty = bx.type_struct(&[bx.type_ptr(), bx.type_i32()], false); |
5e7ed085 FG |
805 | let vals = bx.landing_pad(lpad_ty, bx.eh_personality(), 2); |
806 | bx.add_clause(vals, tydesc); | |
add651ee | 807 | bx.add_clause(vals, bx.const_null(bx.type_ptr())); |
5e7ed085 FG |
808 | let ptr = bx.extract_value(vals, 0); |
809 | let selector = bx.extract_value(vals, 1); | |
1b1a35ee XL |
810 | |
811 | // Check if the typeid we got is the one for a Rust panic. | |
5e7ed085 FG |
812 | let rust_typeid = bx.call_intrinsic("llvm.eh.typeid.for", &[tydesc]); |
813 | let is_rust_panic = bx.icmp(IntPredicate::IntEQ, selector, rust_typeid); | |
814 | let is_rust_panic = bx.zext(is_rust_panic, bx.type_bool()); | |
1b1a35ee XL |
815 | |
816 | // We need to pass two values to catch_func (ptr and is_rust_panic), so | |
817 | // create an alloca and pass a pointer to that. | |
818 | let ptr_align = bx.tcx().data_layout.pointer_align.abi; | |
819 | let i8_align = bx.tcx().data_layout.i8_align.abi; | |
add651ee | 820 | let catch_data_type = bx.type_struct(&[bx.type_ptr(), bx.type_bool()], false); |
5e7ed085 FG |
821 | let catch_data = bx.alloca(catch_data_type, ptr_align); |
822 | let catch_data_0 = | |
823 | bx.inbounds_gep(catch_data_type, catch_data, &[bx.const_usize(0), bx.const_usize(0)]); | |
824 | bx.store(ptr, catch_data_0, ptr_align); | |
825 | let catch_data_1 = | |
826 | bx.inbounds_gep(catch_data_type, catch_data, &[bx.const_usize(0), bx.const_usize(1)]); | |
827 | bx.store(is_rust_panic, catch_data_1, i8_align); | |
1b1a35ee | 828 | |
add651ee | 829 | let catch_ty = bx.type_func(&[bx.type_ptr(), bx.type_ptr()], bx.type_void()); |
49aad941 | 830 | bx.call(catch_ty, None, None, catch_func, &[data, catch_data], None); |
5e7ed085 | 831 | bx.ret(bx.const_i32(1)); |
1b1a35ee XL |
832 | }); |
833 | ||
834 | // Note that no invoke is used here because by definition this function | |
835 | // can't panic (that's what it's catching). | |
49aad941 | 836 | let ret = bx.call(llty, None, None, llfn, &[try_func, data, catch_func], None); |
1b1a35ee XL |
837 | let i32_align = bx.tcx().data_layout.i32_align.abi; |
838 | bx.store(ret, dest, i32_align); | |
839 | } | |
840 | ||
94b46f34 | 841 | // Helper function to give a Block to a closure to codegen a shim function. |
7453a54e | 842 | // This is currently primarily used for the `try` intrinsic functions above. |
b7449926 XL |
843 | fn gen_fn<'ll, 'tcx>( |
844 | cx: &CodegenCx<'ll, 'tcx>, | |
845 | name: &str, | |
29967ef6 | 846 | rust_fn_sig: ty::PolyFnSig<'tcx>, |
b7449926 | 847 | codegen: &mut dyn FnMut(Builder<'_, 'll, 'tcx>), |
94222f64 | 848 | ) -> (&'ll Type, &'ll Value) { |
c295e0f8 | 849 | let fn_abi = cx.fn_abi_of_fn_ptr(rust_fn_sig, ty::List::empty()); |
94222f64 | 850 | let llty = fn_abi.llvm_type(cx); |
49aad941 | 851 | let llfn = cx.declare_fn(name, fn_abi, None); |
136023e0 | 852 | cx.set_frame_pointer_type(llfn); |
ba9703b0 | 853 | cx.apply_target_cpu_attr(llfn); |
60c5eb7d XL |
854 | // FIXME(eddyb) find a nicer way to do this. |
855 | unsafe { llvm::LLVMRustSetLinkage(llfn, llvm::Linkage::InternalLinkage) }; | |
17df50a5 XL |
856 | let llbb = Builder::append_block(cx, llfn, "entry-block"); |
857 | let bx = Builder::build(cx, llbb); | |
94b46f34 | 858 | codegen(bx); |
94222f64 | 859 | (llty, llfn) |
7453a54e SL |
860 | } |
861 | ||
862 | // Helper function used to get a handle to the `__rust_try` function used to | |
863 | // catch exceptions. | |
864 | // | |
865 | // This function is only generated once and is then cached. | |
b7449926 XL |
866 | fn get_rust_try_fn<'ll, 'tcx>( |
867 | cx: &CodegenCx<'ll, 'tcx>, | |
868 | codegen: &mut dyn FnMut(Builder<'_, 'll, 'tcx>), | |
94222f64 | 869 | ) -> (&'ll Type, &'ll Value) { |
2c00a5a8 | 870 | if let Some(llfn) = cx.rust_try_fn.get() { |
54a0048b | 871 | return llfn; |
c1a9b12d SL |
872 | } |
873 | ||
874 | // Define the type up front for the signature of the rust_try function. | |
2c00a5a8 | 875 | let tcx = cx.tcx; |
fe692bf9 | 876 | let i8p = Ty::new_mut_ptr(tcx, tcx.types.i8); |
29967ef6 | 877 | // `unsafe fn(*mut i8) -> ()` |
fe692bf9 FG |
878 | let try_fn_ty = Ty::new_fn_ptr( |
879 | tcx, | |
880 | ty::Binder::dummy(tcx.mk_fn_sig( | |
881 | [i8p], | |
882 | Ty::new_unit(tcx), | |
883 | false, | |
884 | hir::Unsafety::Unsafe, | |
885 | Abi::Rust, | |
886 | )), | |
887 | ); | |
29967ef6 | 888 | // `unsafe fn(*mut i8, *mut i8) -> ()` |
fe692bf9 FG |
889 | let catch_fn_ty = Ty::new_fn_ptr( |
890 | tcx, | |
891 | ty::Binder::dummy(tcx.mk_fn_sig( | |
892 | [i8p, i8p], | |
893 | Ty::new_unit(tcx), | |
894 | false, | |
895 | hir::Unsafety::Unsafe, | |
896 | Abi::Rust, | |
897 | )), | |
898 | ); | |
29967ef6 XL |
899 | // `unsafe fn(unsafe fn(*mut i8) -> (), *mut i8, unsafe fn(*mut i8, *mut i8) -> ()) -> i32` |
900 | let rust_fn_sig = ty::Binder::dummy(cx.tcx.mk_fn_sig( | |
9ffffee4 | 901 | [try_fn_ty, i8p, catch_fn_ty], |
29967ef6 XL |
902 | tcx.types.i32, |
903 | false, | |
904 | hir::Unsafety::Unsafe, | |
905 | Abi::Rust, | |
906 | )); | |
907 | let rust_try = gen_fn(cx, "__rust_try", rust_fn_sig, codegen); | |
2c00a5a8 | 908 | cx.rust_try_fn.set(Some(rust_try)); |
0bf4aa26 | 909 | rust_try |
c1a9b12d | 910 | } |
e9174d1e | 911 | |
a2a8927a XL |
912 | fn generic_simd_intrinsic<'ll, 'tcx>( |
913 | bx: &mut Builder<'_, 'll, 'tcx>, | |
3dfed10e | 914 | name: Symbol, |
32a655c1 | 915 | callee_ty: Ty<'tcx>, |
781aab86 | 916 | fn_args: GenericArgsRef<'tcx>, |
a1dfa0c6 | 917 | args: &[OperandRef<'tcx, &'ll Value>], |
32a655c1 | 918 | ret_ty: Ty<'tcx>, |
b7449926 | 919 | llret_ty: &'ll Type, |
dfeec247 | 920 | span: Span, |
b7449926 | 921 | ) -> Result<&'ll Value, ()> { |
0531ce1d | 922 | macro_rules! return_error { |
9c376795 FG |
923 | ($diag: expr) => {{ |
924 | bx.sess().emit_err($diag); | |
925 | return Err(()); | |
926 | }}; | |
e9174d1e | 927 | } |
0531ce1d XL |
928 | |
929 | macro_rules! require { | |
9c376795 | 930 | ($cond: expr, $diag: expr) => { |
0531ce1d | 931 | if !$cond { |
9c376795 | 932 | return_error!($diag); |
0531ce1d XL |
933 | } |
934 | }; | |
935 | } | |
0bf4aa26 | 936 | |
e9174d1e | 937 | macro_rules! require_simd { |
9c376795 FG |
938 | ($ty: expr, $diag: expr) => { |
939 | require!($ty.is_simd(), $diag) | |
dfeec247 | 940 | }; |
e9174d1e SL |
941 | } |
942 | ||
2c00a5a8 | 943 | let tcx = bx.tcx(); |
fc512014 XL |
944 | let sig = |
945 | tcx.normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), callee_ty.fn_sig(tcx)); | |
476ff2be | 946 | let arg_tys = sig.inputs(); |
e9174d1e | 947 | |
3dfed10e | 948 | if name == sym::simd_select_bitmask { |
9c376795 FG |
949 | require_simd!( |
950 | arg_tys[1], | |
951 | InvalidMonomorphization::SimdArgument { span, name, ty: arg_tys[1] } | |
952 | ); | |
953 | ||
3c0e092e XL |
954 | let (len, _) = arg_tys[1].simd_size_and_type(bx.tcx()); |
955 | ||
956 | let expected_int_bits = (len.max(8) - 1).next_power_of_two(); | |
957 | let expected_bytes = len / 8 + ((len % 8 > 0) as u64); | |
958 | ||
959 | let mask_ty = arg_tys[0]; | |
960 | let mask = match mask_ty.kind() { | |
961 | ty::Int(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(), | |
962 | ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(), | |
963 | ty::Array(elem, len) | |
964 | if matches!(elem.kind(), ty::Uint(ty::UintTy::U8)) | |
9ffffee4 | 965 | && len.try_eval_target_usize(bx.tcx, ty::ParamEnv::reveal_all()) |
3c0e092e XL |
966 | == Some(expected_bytes) => |
967 | { | |
968 | let place = PlaceRef::alloca(bx, args[0].layout); | |
969 | args[0].val.store(bx, place); | |
970 | let int_ty = bx.type_ix(expected_bytes * 8); | |
add651ee | 971 | bx.load(int_ty, place.llval, Align::ONE) |
3c0e092e | 972 | } |
9c376795 FG |
973 | _ => return_error!(InvalidMonomorphization::InvalidBitmask { |
974 | span, | |
975 | name, | |
3c0e092e XL |
976 | mask_ty, |
977 | expected_int_bits, | |
978 | expected_bytes | |
9c376795 | 979 | }), |
3c0e092e XL |
980 | }; |
981 | ||
0731742a | 982 | let i1 = bx.type_i1(); |
3c0e092e XL |
983 | let im = bx.type_ix(len); |
984 | let i1xn = bx.type_vector(i1, len); | |
985 | let m_im = bx.trunc(mask, im); | |
29967ef6 | 986 | let m_i1s = bx.bitcast(m_im, i1xn); |
0731742a XL |
987 | return Ok(bx.select(m_i1s, args[1].immediate(), args[2].immediate())); |
988 | } | |
989 | ||
990 | // every intrinsic below takes a SIMD vector as its first argument | |
9c376795 | 991 | require_simd!(arg_tys[0], InvalidMonomorphization::SimdInput { span, name, ty: arg_tys[0] }); |
e9174d1e | 992 | let in_ty = arg_tys[0]; |
e9174d1e SL |
993 | |
994 | let comparison = match name { | |
3dfed10e XL |
995 | sym::simd_eq => Some(hir::BinOpKind::Eq), |
996 | sym::simd_ne => Some(hir::BinOpKind::Ne), | |
997 | sym::simd_lt => Some(hir::BinOpKind::Lt), | |
998 | sym::simd_le => Some(hir::BinOpKind::Le), | |
999 | sym::simd_gt => Some(hir::BinOpKind::Gt), | |
1000 | sym::simd_ge => Some(hir::BinOpKind::Ge), | |
dfeec247 | 1001 | _ => None, |
e9174d1e SL |
1002 | }; |
1003 | ||
fc512014 | 1004 | let (in_len, in_elem) = arg_tys[0].simd_size_and_type(bx.tcx()); |
e9174d1e | 1005 | if let Some(cmp_op) = comparison { |
9c376795 | 1006 | require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty }); |
e9174d1e | 1007 | |
fc512014 | 1008 | let (out_len, out_ty) = ret_ty.simd_size_and_type(bx.tcx()); |
9c376795 | 1009 | |
dfeec247 XL |
1010 | require!( |
1011 | in_len == out_len, | |
9c376795 FG |
1012 | InvalidMonomorphization::ReturnLengthInputType { |
1013 | span, | |
1014 | name, | |
1015 | in_len, | |
1016 | in_ty, | |
1017 | ret_ty, | |
1018 | out_len | |
1019 | } | |
dfeec247 XL |
1020 | ); |
1021 | require!( | |
1022 | bx.type_kind(bx.element_type(llret_ty)) == TypeKind::Integer, | |
9c376795 | 1023 | InvalidMonomorphization::ReturnIntegerType { span, name, ret_ty, out_ty } |
dfeec247 XL |
1024 | ); |
1025 | ||
1026 | return Ok(compare_simd_types( | |
1027 | bx, | |
1028 | args[0].immediate(), | |
1029 | args[1].immediate(), | |
1030 | in_elem, | |
1031 | llret_ty, | |
1032 | cmp_op, | |
1033 | )); | |
e9174d1e SL |
1034 | } |
1035 | ||
781aab86 FG |
1036 | if name == sym::simd_shuffle_generic { |
1037 | let idx = fn_args[2] | |
1038 | .expect_const() | |
1039 | .eval(tcx, ty::ParamEnv::reveal_all(), Some(span)) | |
1040 | .unwrap() | |
1041 | .unwrap_branch(); | |
1042 | let n = idx.len() as u64; | |
1043 | ||
1044 | require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty }); | |
1045 | let (out_len, out_ty) = ret_ty.simd_size_and_type(bx.tcx()); | |
1046 | require!( | |
1047 | out_len == n, | |
1048 | InvalidMonomorphization::ReturnLength { span, name, in_len: n, ret_ty, out_len } | |
1049 | ); | |
1050 | require!( | |
1051 | in_elem == out_ty, | |
1052 | InvalidMonomorphization::ReturnElement { span, name, in_elem, in_ty, ret_ty, out_ty } | |
1053 | ); | |
1054 | ||
1055 | let total_len = in_len * 2; | |
1056 | ||
1057 | let indices: Option<Vec<_>> = idx | |
1058 | .iter() | |
1059 | .enumerate() | |
1060 | .map(|(arg_idx, val)| { | |
1061 | let idx = val.unwrap_leaf().try_to_i32().unwrap(); | |
1062 | if idx >= i32::try_from(total_len).unwrap() { | |
1063 | bx.sess().emit_err(InvalidMonomorphization::ShuffleIndexOutOfBounds { | |
1064 | span, | |
1065 | name, | |
1066 | arg_idx: arg_idx as u64, | |
1067 | total_len: total_len.into(), | |
1068 | }); | |
1069 | None | |
1070 | } else { | |
1071 | Some(bx.const_i32(idx)) | |
1072 | } | |
1073 | }) | |
1074 | .collect(); | |
1075 | let Some(indices) = indices else { | |
1076 | return Ok(bx.const_null(llret_ty)); | |
1077 | }; | |
1078 | ||
1079 | return Ok(bx.shuffle_vector( | |
1080 | args[0].immediate(), | |
1081 | args[1].immediate(), | |
1082 | bx.const_vector(&indices), | |
1083 | )); | |
1084 | } | |
1085 | ||
add651ee FG |
1086 | if name == sym::simd_shuffle { |
1087 | // Make sure this is actually an array, since typeck only checks the length-suffixed | |
1088 | // version of this intrinsic. | |
1089 | let n: u64 = match args[2].layout.ty.kind() { | |
1090 | ty::Array(ty, len) if matches!(ty.kind(), ty::Uint(ty::UintTy::U32)) => { | |
1091 | len.try_eval_target_usize(bx.cx.tcx, ty::ParamEnv::reveal_all()).unwrap_or_else( | |
1092 | || span_bug!(span, "could not evaluate shuffle index array length"), | |
1093 | ) | |
c295e0f8 | 1094 | } |
add651ee FG |
1095 | _ => return_error!(InvalidMonomorphization::SimdShuffle { |
1096 | span, | |
1097 | name, | |
1098 | ty: args[2].layout.ty | |
1099 | }), | |
c295e0f8 | 1100 | }; |
e9174d1e | 1101 | |
9c376795 | 1102 | require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty }); |
fc512014 | 1103 | let (out_len, out_ty) = ret_ty.simd_size_and_type(bx.tcx()); |
dfeec247 XL |
1104 | require!( |
1105 | out_len == n, | |
9c376795 | 1106 | InvalidMonomorphization::ReturnLength { span, name, in_len: n, ret_ty, out_len } |
dfeec247 XL |
1107 | ); |
1108 | require!( | |
fc512014 | 1109 | in_elem == out_ty, |
9c376795 | 1110 | InvalidMonomorphization::ReturnElement { span, name, in_elem, in_ty, ret_ty, out_ty } |
dfeec247 | 1111 | ); |
e9174d1e | 1112 | |
60c5eb7d | 1113 | let total_len = u128::from(in_len) * 2; |
e9174d1e | 1114 | |
ff7c6d11 | 1115 | let vector = args[2].immediate(); |
e9174d1e SL |
1116 | |
1117 | let indices: Option<Vec<_>> = (0..n) | |
1118 | .map(|i| { | |
1119 | let arg_idx = i; | |
a1dfa0c6 XL |
1120 | let val = bx.const_get_elt(vector, i as u64); |
1121 | match bx.const_to_opt_u128(val, true) { | |
e9174d1e | 1122 | None => { |
9c376795 FG |
1123 | bx.sess().emit_err(InvalidMonomorphization::ShuffleIndexNotConstant { |
1124 | span, | |
1125 | name, | |
1126 | arg_idx, | |
1127 | }); | |
e9174d1e SL |
1128 | None |
1129 | } | |
1130 | Some(idx) if idx >= total_len => { | |
9c376795 FG |
1131 | bx.sess().emit_err(InvalidMonomorphization::ShuffleIndexOutOfBounds { |
1132 | span, | |
1133 | name, | |
dfeec247 | 1134 | arg_idx, |
9c376795 FG |
1135 | total_len, |
1136 | }); | |
e9174d1e SL |
1137 | None |
1138 | } | |
a1dfa0c6 | 1139 | Some(idx) => Some(bx.const_i32(idx as i32)), |
e9174d1e SL |
1140 | } |
1141 | }) | |
1142 | .collect(); | |
5e7ed085 FG |
1143 | let Some(indices) = indices else { |
1144 | return Ok(bx.const_null(llret_ty)); | |
e9174d1e SL |
1145 | }; |
1146 | ||
dfeec247 XL |
1147 | return Ok(bx.shuffle_vector( |
1148 | args[0].immediate(), | |
1149 | args[1].immediate(), | |
1150 | bx.const_vector(&indices), | |
1151 | )); | |
e9174d1e SL |
1152 | } |
1153 | ||
3dfed10e | 1154 | if name == sym::simd_insert { |
dfeec247 XL |
1155 | require!( |
1156 | in_elem == arg_tys[2], | |
9c376795 FG |
1157 | InvalidMonomorphization::InsertedType { |
1158 | span, | |
1159 | name, | |
1160 | in_elem, | |
1161 | in_ty, | |
1162 | out_ty: arg_tys[2] | |
1163 | } | |
dfeec247 XL |
1164 | ); |
1165 | return Ok(bx.insert_element( | |
1166 | args[0].immediate(), | |
1167 | args[2].immediate(), | |
1168 | args[1].immediate(), | |
1169 | )); | |
e9174d1e | 1170 | } |
3dfed10e | 1171 | if name == sym::simd_extract { |
dfeec247 XL |
1172 | require!( |
1173 | ret_ty == in_elem, | |
9c376795 | 1174 | InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty } |
dfeec247 XL |
1175 | ); |
1176 | return Ok(bx.extract_element(args[0].immediate(), args[1].immediate())); | |
e9174d1e SL |
1177 | } |
1178 | ||
3dfed10e | 1179 | if name == sym::simd_select { |
0531ce1d XL |
1180 | let m_elem_ty = in_elem; |
1181 | let m_len = in_len; | |
9c376795 FG |
1182 | require_simd!( |
1183 | arg_tys[1], | |
1184 | InvalidMonomorphization::SimdArgument { span, name, ty: arg_tys[1] } | |
1185 | ); | |
fc512014 | 1186 | let (v_len, _) = arg_tys[1].simd_size_and_type(bx.tcx()); |
dfeec247 XL |
1187 | require!( |
1188 | m_len == v_len, | |
9c376795 | 1189 | InvalidMonomorphization::MismatchedLengths { span, name, m_len, v_len } |
0531ce1d | 1190 | ); |
1b1a35ee | 1191 | match m_elem_ty.kind() { |
dfeec247 | 1192 | ty::Int(_) => {} |
9c376795 | 1193 | _ => return_error!(InvalidMonomorphization::MaskType { span, name, ty: m_elem_ty }), |
0531ce1d XL |
1194 | } |
1195 | // truncate the mask to a vector of i1s | |
a1dfa0c6 XL |
1196 | let i1 = bx.type_i1(); |
1197 | let i1xn = bx.type_vector(i1, m_len as u64); | |
0531ce1d XL |
1198 | let m_i1s = bx.trunc(args[0].immediate(), i1xn); |
1199 | return Ok(bx.select(m_i1s, args[1].immediate(), args[2].immediate())); | |
1200 | } | |
1201 | ||
3dfed10e | 1202 | if name == sym::simd_bitmask { |
9fa01778 | 1203 | // The `fn simd_bitmask(vector) -> unsigned integer` intrinsic takes a |
3c0e092e XL |
1204 | // vector mask and returns the most significant bit (MSB) of each lane in the form |
1205 | // of either: | |
1206 | // * an unsigned integer | |
1207 | // * an array of `u8` | |
1208 | // If the vector has less than 8 lanes, a u8 is returned with zeroed trailing bits. | |
1209 | // | |
1210 | // The bit order of the result depends on the byte endianness, LSB-first for little | |
1211 | // endian and MSB-first for big endian. | |
9fa01778 | 1212 | let expected_int_bits = in_len.max(8); |
3c0e092e | 1213 | let expected_bytes = expected_int_bits / 8 + ((expected_int_bits % 8 > 0) as u64); |
9fa01778 XL |
1214 | |
1215 | // Integer vector <i{in_bitwidth} x in_len>: | |
1b1a35ee | 1216 | let (i_xn, in_elem_bitwidth) = match in_elem.kind() { |
29967ef6 XL |
1217 | ty::Int(i) => ( |
1218 | args[0].immediate(), | |
1219 | i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size.bits()), | |
1220 | ), | |
1221 | ty::Uint(i) => ( | |
1222 | args[0].immediate(), | |
1223 | i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size.bits()), | |
1224 | ), | |
9c376795 FG |
1225 | _ => return_error!(InvalidMonomorphization::VectorArgument { |
1226 | span, | |
1227 | name, | |
dfeec247 XL |
1228 | in_ty, |
1229 | in_elem | |
9c376795 | 1230 | }), |
9fa01778 XL |
1231 | }; |
1232 | ||
1233 | // Shift the MSB to the right by "in_elem_bitwidth - 1" into the first bit position. | |
dfeec247 XL |
1234 | let shift_indices = |
1235 | vec![ | |
ba9703b0 | 1236 | bx.cx.const_int(bx.type_ix(in_elem_bitwidth), (in_elem_bitwidth - 1) as _); |
dfeec247 XL |
1237 | in_len as _ |
1238 | ]; | |
9fa01778 XL |
1239 | let i_xn_msb = bx.lshr(i_xn, bx.const_vector(shift_indices.as_slice())); |
1240 | // Truncate vector to an <i1 x N> | |
ba9703b0 | 1241 | let i1xn = bx.trunc(i_xn_msb, bx.type_vector(bx.type_i1(), in_len)); |
9fa01778 | 1242 | // Bitcast <i1 x N> to iN: |
ba9703b0 | 1243 | let i_ = bx.bitcast(i1xn, bx.type_ix(in_len)); |
3c0e092e XL |
1244 | |
1245 | match ret_ty.kind() { | |
1246 | ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => { | |
1247 | // Zero-extend iN to the bitmask type: | |
1248 | return Ok(bx.zext(i_, bx.type_ix(expected_int_bits))); | |
1249 | } | |
1250 | ty::Array(elem, len) | |
1251 | if matches!(elem.kind(), ty::Uint(ty::UintTy::U8)) | |
9ffffee4 | 1252 | && len.try_eval_target_usize(bx.tcx, ty::ParamEnv::reveal_all()) |
3c0e092e XL |
1253 | == Some(expected_bytes) => |
1254 | { | |
5e7ed085 | 1255 | // Zero-extend iN to the array length: |
3c0e092e XL |
1256 | let ze = bx.zext(i_, bx.type_ix(expected_bytes * 8)); |
1257 | ||
1258 | // Convert the integer to a byte array | |
1259 | let ptr = bx.alloca(bx.type_ix(expected_bytes * 8), Align::ONE); | |
1260 | bx.store(ze, ptr, Align::ONE); | |
1261 | let array_ty = bx.type_array(bx.type_i8(), expected_bytes); | |
3c0e092e XL |
1262 | return Ok(bx.load(array_ty, ptr, Align::ONE)); |
1263 | } | |
9c376795 FG |
1264 | _ => return_error!(InvalidMonomorphization::CannotReturn { |
1265 | span, | |
1266 | name, | |
3c0e092e XL |
1267 | ret_ty, |
1268 | expected_int_bits, | |
1269 | expected_bytes | |
9c376795 | 1270 | }), |
3c0e092e | 1271 | } |
9fa01778 XL |
1272 | } |
1273 | ||
a2a8927a | 1274 | fn simd_simple_float_intrinsic<'ll, 'tcx>( |
6a06907d | 1275 | name: Symbol, |
5099ac24 FG |
1276 | in_elem: Ty<'_>, |
1277 | in_ty: Ty<'_>, | |
60c5eb7d | 1278 | in_len: u64, |
a2a8927a | 1279 | bx: &mut Builder<'_, 'll, 'tcx>, |
b7449926 | 1280 | span: Span, |
a1dfa0c6 | 1281 | args: &[OperandRef<'tcx, &'ll Value>], |
b7449926 | 1282 | ) -> Result<&'ll Value, ()> { |
94b46f34 | 1283 | macro_rules! return_error { |
9c376795 FG |
1284 | ($diag: expr) => {{ |
1285 | bx.sess().emit_err($diag); | |
1286 | return Err(()); | |
1287 | }}; | |
94b46f34 | 1288 | } |
6a06907d XL |
1289 | |
1290 | let (elem_ty_str, elem_ty) = if let ty::Float(f) = in_elem.kind() { | |
1291 | let elem_ty = bx.cx.type_float_from_ty(*f); | |
1292 | match f.bit_width() { | |
1293 | 32 => ("f32", elem_ty), | |
1294 | 64 => ("f64", elem_ty), | |
9c376795 FG |
1295 | _ => return_error!(InvalidMonomorphization::FloatingPointVector { |
1296 | span, | |
1297 | name, | |
1298 | f_ty: *f, | |
1299 | in_ty, | |
1300 | }), | |
94b46f34 | 1301 | } |
6a06907d | 1302 | } else { |
9c376795 | 1303 | return_error!(InvalidMonomorphization::FloatingPointType { span, name, in_ty }); |
94b46f34 XL |
1304 | }; |
1305 | ||
6a06907d XL |
1306 | let vec_ty = bx.type_vector(elem_ty, in_len); |
1307 | ||
1308 | let (intr_name, fn_ty) = match name { | |
6a06907d | 1309 | sym::simd_ceil => ("ceil", bx.type_func(&[vec_ty], vec_ty)), |
cdc7bbd5 XL |
1310 | sym::simd_fabs => ("fabs", bx.type_func(&[vec_ty], vec_ty)), |
1311 | sym::simd_fcos => ("cos", bx.type_func(&[vec_ty], vec_ty)), | |
6a06907d | 1312 | sym::simd_fexp2 => ("exp2", bx.type_func(&[vec_ty], vec_ty)), |
cdc7bbd5 | 1313 | sym::simd_fexp => ("exp", bx.type_func(&[vec_ty], vec_ty)), |
6a06907d XL |
1314 | sym::simd_flog10 => ("log10", bx.type_func(&[vec_ty], vec_ty)), |
1315 | sym::simd_flog2 => ("log2", bx.type_func(&[vec_ty], vec_ty)), | |
1316 | sym::simd_flog => ("log", bx.type_func(&[vec_ty], vec_ty)), | |
cdc7bbd5 XL |
1317 | sym::simd_floor => ("floor", bx.type_func(&[vec_ty], vec_ty)), |
1318 | sym::simd_fma => ("fma", bx.type_func(&[vec_ty, vec_ty, vec_ty], vec_ty)), | |
6a06907d XL |
1319 | sym::simd_fpowi => ("powi", bx.type_func(&[vec_ty, bx.type_i32()], vec_ty)), |
1320 | sym::simd_fpow => ("pow", bx.type_func(&[vec_ty, vec_ty], vec_ty)), | |
cdc7bbd5 XL |
1321 | sym::simd_fsin => ("sin", bx.type_func(&[vec_ty], vec_ty)), |
1322 | sym::simd_fsqrt => ("sqrt", bx.type_func(&[vec_ty], vec_ty)), | |
1323 | sym::simd_round => ("round", bx.type_func(&[vec_ty], vec_ty)), | |
1324 | sym::simd_trunc => ("trunc", bx.type_func(&[vec_ty], vec_ty)), | |
9c376795 | 1325 | _ => return_error!(InvalidMonomorphization::UnrecognizedIntrinsic { span, name }), |
6a06907d | 1326 | }; |
add651ee | 1327 | let llvm_name = &format!("llvm.{intr_name}.v{in_len}{elem_ty_str}"); |
c295e0f8 | 1328 | let f = bx.declare_cfn(llvm_name, llvm::UnnamedAddr::No, fn_ty); |
2b03887a FG |
1329 | let c = bx.call( |
1330 | fn_ty, | |
1331 | None, | |
49aad941 | 1332 | None, |
2b03887a FG |
1333 | f, |
1334 | &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(), | |
1335 | None, | |
1336 | ); | |
0bf4aa26 | 1337 | Ok(c) |
94b46f34 XL |
1338 | } |
1339 | ||
6a06907d XL |
1340 | if std::matches!( |
1341 | name, | |
cdc7bbd5 | 1342 | sym::simd_ceil |
6a06907d | 1343 | | sym::simd_fabs |
cdc7bbd5 | 1344 | | sym::simd_fcos |
6a06907d | 1345 | | sym::simd_fexp2 |
cdc7bbd5 | 1346 | | sym::simd_fexp |
6a06907d XL |
1347 | | sym::simd_flog10 |
1348 | | sym::simd_flog2 | |
1349 | | sym::simd_flog | |
cdc7bbd5 | 1350 | | sym::simd_floor |
6a06907d | 1351 | | sym::simd_fma |
cdc7bbd5 XL |
1352 | | sym::simd_fpow |
1353 | | sym::simd_fpowi | |
1354 | | sym::simd_fsin | |
1355 | | sym::simd_fsqrt | |
1356 | | sym::simd_round | |
1357 | | sym::simd_trunc | |
6a06907d XL |
1358 | ) { |
1359 | return simd_simple_float_intrinsic(name, in_elem, in_ty, in_len, bx, span, args); | |
94b46f34 XL |
1360 | } |
1361 | ||
1362 | // FIXME: use: | |
1363 | // https://github.com/llvm-mirror/llvm/blob/master/include/llvm/IR/Function.h#L182 | |
1364 | // https://github.com/llvm-mirror/llvm/blob/master/include/llvm/IR/Intrinsics.h#L81 | |
add651ee | 1365 | fn llvm_vector_str(bx: &Builder<'_, '_, '_>, elem_ty: Ty<'_>, vec_len: u64) -> String { |
1b1a35ee | 1366 | match *elem_ty.kind() { |
c295e0f8 | 1367 | ty::Int(v) => format!( |
add651ee | 1368 | "v{}i{}", |
c295e0f8 | 1369 | vec_len, |
c295e0f8 XL |
1370 | // Normalize to prevent crash if v: IntTy::Isize |
1371 | v.normalize(bx.target_spec().pointer_width).bit_width().unwrap() | |
1372 | ), | |
1373 | ty::Uint(v) => format!( | |
add651ee | 1374 | "v{}i{}", |
c295e0f8 | 1375 | vec_len, |
c295e0f8 XL |
1376 | // Normalize to prevent crash if v: UIntTy::Usize |
1377 | v.normalize(bx.target_spec().pointer_width).bit_width().unwrap() | |
1378 | ), | |
add651ee FG |
1379 | ty::Float(v) => format!("v{}f{}", vec_len, v.bit_width()), |
1380 | ty::RawPtr(_) => format!("v{}p0", vec_len), | |
94b46f34 XL |
1381 | _ => unreachable!(), |
1382 | } | |
1383 | } | |
1384 | ||
add651ee FG |
1385 | fn llvm_vector_ty<'ll>(cx: &CodegenCx<'ll, '_>, elem_ty: Ty<'_>, vec_len: u64) -> &'ll Type { |
1386 | let elem_ty = match *elem_ty.kind() { | |
dfeec247 XL |
1387 | ty::Int(v) => cx.type_int_from_ty(v), |
1388 | ty::Uint(v) => cx.type_uint_from_ty(v), | |
1389 | ty::Float(v) => cx.type_float_from_ty(v), | |
add651ee | 1390 | ty::RawPtr(_) => cx.type_ptr(), |
94b46f34 XL |
1391 | _ => unreachable!(), |
1392 | }; | |
60c5eb7d | 1393 | cx.type_vector(elem_ty, vec_len) |
94b46f34 XL |
1394 | } |
1395 | ||
3dfed10e | 1396 | if name == sym::simd_gather { |
94b46f34 XL |
1397 | // simd_gather(values: <N x T>, pointers: <N x *_ T>, |
1398 | // mask: <N x i{M}>) -> <N x T> | |
1399 | // * N: number of elements in the input vectors | |
1400 | // * T: type of the element to load | |
1401 | // * M: any integer width is supported, will be truncated to i1 | |
1402 | ||
1403 | // All types must be simd vector types | |
9c376795 FG |
1404 | require_simd!(in_ty, InvalidMonomorphization::SimdFirst { span, name, ty: in_ty }); |
1405 | require_simd!( | |
1406 | arg_tys[1], | |
1407 | InvalidMonomorphization::SimdSecond { span, name, ty: arg_tys[1] } | |
1408 | ); | |
1409 | require_simd!( | |
1410 | arg_tys[2], | |
1411 | InvalidMonomorphization::SimdThird { span, name, ty: arg_tys[2] } | |
1412 | ); | |
1413 | require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty }); | |
94b46f34 XL |
1414 | |
1415 | // Of the same length: | |
fc512014 XL |
1416 | let (out_len, _) = arg_tys[1].simd_size_and_type(bx.tcx()); |
1417 | let (out_len2, _) = arg_tys[2].simd_size_and_type(bx.tcx()); | |
dfeec247 | 1418 | require!( |
fc512014 | 1419 | in_len == out_len, |
9c376795 FG |
1420 | InvalidMonomorphization::SecondArgumentLength { |
1421 | span, | |
1422 | name, | |
1423 | in_len, | |
1424 | in_ty, | |
1425 | arg_ty: arg_tys[1], | |
1426 | out_len | |
1427 | } | |
dfeec247 XL |
1428 | ); |
1429 | require!( | |
fc512014 | 1430 | in_len == out_len2, |
9c376795 FG |
1431 | InvalidMonomorphization::ThirdArgumentLength { |
1432 | span, | |
1433 | name, | |
1434 | in_len, | |
1435 | in_ty, | |
1436 | arg_ty: arg_tys[2], | |
1437 | out_len: out_len2 | |
1438 | } | |
dfeec247 | 1439 | ); |
94b46f34 XL |
1440 | |
1441 | // The return type must match the first argument type | |
9c376795 FG |
1442 | require!( |
1443 | ret_ty == in_ty, | |
1444 | InvalidMonomorphization::ExpectedReturnType { span, name, in_ty, ret_ty } | |
1445 | ); | |
94b46f34 | 1446 | |
94b46f34 XL |
1447 | // The second argument must be a simd vector with an element type that's a pointer |
1448 | // to the element type of the first argument | |
fc512014 XL |
1449 | let (_, element_ty0) = arg_tys[0].simd_size_and_type(bx.tcx()); |
1450 | let (_, element_ty1) = arg_tys[1].simd_size_and_type(bx.tcx()); | |
add651ee FG |
1451 | |
1452 | require!( | |
1453 | matches!( | |
1454 | element_ty1.kind(), | |
1455 | ty::RawPtr(p) if p.ty == in_elem && p.ty.kind() == element_ty0.kind() | |
1456 | ), | |
1457 | InvalidMonomorphization::ExpectedElementType { | |
1458 | span, | |
1459 | name, | |
1460 | expected_element: element_ty1, | |
1461 | second_arg: arg_tys[1], | |
1462 | in_elem, | |
1463 | in_ty, | |
1464 | mutability: ExpectedPointerMutability::Not, | |
94b46f34 | 1465 | } |
add651ee | 1466 | ); |
94b46f34 XL |
1467 | |
1468 | // The element type of the third argument must be a signed integer type of any width: | |
fc512014 XL |
1469 | let (_, element_ty2) = arg_tys[2].simd_size_and_type(bx.tcx()); |
1470 | match element_ty2.kind() { | |
b7449926 | 1471 | ty::Int(_) => (), |
94b46f34 | 1472 | _ => { |
dfeec247 XL |
1473 | require!( |
1474 | false, | |
9c376795 FG |
1475 | InvalidMonomorphization::ThirdArgElementType { |
1476 | span, | |
1477 | name, | |
1478 | expected_element: element_ty2, | |
1479 | third_arg: arg_tys[2] | |
1480 | } | |
dfeec247 | 1481 | ); |
94b46f34 XL |
1482 | } |
1483 | } | |
1484 | ||
1485 | // Alignment of T, must be a constant integer value: | |
a1dfa0c6 XL |
1486 | let alignment_ty = bx.type_i32(); |
1487 | let alignment = bx.const_i32(bx.align_of(in_elem).bytes() as i32); | |
94b46f34 XL |
1488 | |
1489 | // Truncate the mask vector to a vector of i1s: | |
1490 | let (mask, mask_ty) = { | |
a1dfa0c6 | 1491 | let i1 = bx.type_i1(); |
60c5eb7d | 1492 | let i1xn = bx.type_vector(i1, in_len); |
94b46f34 XL |
1493 | (bx.trunc(args[2].immediate(), i1xn), i1xn) |
1494 | }; | |
1495 | ||
1496 | // Type of the vector of pointers: | |
add651ee FG |
1497 | let llvm_pointer_vec_ty = llvm_vector_ty(bx, element_ty1, in_len); |
1498 | let llvm_pointer_vec_str = llvm_vector_str(bx, element_ty1, in_len); | |
94b46f34 XL |
1499 | |
1500 | // Type of the vector of elements: | |
add651ee FG |
1501 | let llvm_elem_vec_ty = llvm_vector_ty(bx, element_ty0, in_len); |
1502 | let llvm_elem_vec_str = llvm_vector_str(bx, element_ty0, in_len); | |
94b46f34 | 1503 | |
dfeec247 | 1504 | let llvm_intrinsic = |
add651ee | 1505 | format!("llvm.masked.gather.{llvm_elem_vec_str}.{llvm_pointer_vec_str}"); |
94222f64 XL |
1506 | let fn_ty = bx.type_func( |
1507 | &[llvm_pointer_vec_ty, alignment_ty, mask_ty, llvm_elem_vec_ty], | |
1508 | llvm_elem_vec_ty, | |
dfeec247 | 1509 | ); |
94222f64 | 1510 | let f = bx.declare_cfn(&llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); |
2b03887a FG |
1511 | let v = bx.call( |
1512 | fn_ty, | |
1513 | None, | |
49aad941 | 1514 | None, |
2b03887a FG |
1515 | f, |
1516 | &[args[1].immediate(), alignment, mask, args[0].immediate()], | |
1517 | None, | |
1518 | ); | |
94b46f34 XL |
1519 | return Ok(v); |
1520 | } | |
1521 | ||
3dfed10e | 1522 | if name == sym::simd_scatter { |
94b46f34 XL |
1523 | // simd_scatter(values: <N x T>, pointers: <N x *mut T>, |
1524 | // mask: <N x i{M}>) -> () | |
1525 | // * N: number of elements in the input vectors | |
1526 | // * T: type of the element to load | |
1527 | // * M: any integer width is supported, will be truncated to i1 | |
1528 | ||
1529 | // All types must be simd vector types | |
9c376795 FG |
1530 | require_simd!(in_ty, InvalidMonomorphization::SimdFirst { span, name, ty: in_ty }); |
1531 | require_simd!( | |
1532 | arg_tys[1], | |
1533 | InvalidMonomorphization::SimdSecond { span, name, ty: arg_tys[1] } | |
1534 | ); | |
1535 | require_simd!( | |
1536 | arg_tys[2], | |
1537 | InvalidMonomorphization::SimdThird { span, name, ty: arg_tys[2] } | |
1538 | ); | |
94b46f34 XL |
1539 | |
1540 | // Of the same length: | |
fc512014 XL |
1541 | let (element_len1, _) = arg_tys[1].simd_size_and_type(bx.tcx()); |
1542 | let (element_len2, _) = arg_tys[2].simd_size_and_type(bx.tcx()); | |
dfeec247 | 1543 | require!( |
fc512014 | 1544 | in_len == element_len1, |
9c376795 FG |
1545 | InvalidMonomorphization::SecondArgumentLength { |
1546 | span, | |
1547 | name, | |
1548 | in_len, | |
1549 | in_ty, | |
1550 | arg_ty: arg_tys[1], | |
1551 | out_len: element_len1 | |
1552 | } | |
dfeec247 XL |
1553 | ); |
1554 | require!( | |
fc512014 | 1555 | in_len == element_len2, |
9c376795 FG |
1556 | InvalidMonomorphization::ThirdArgumentLength { |
1557 | span, | |
1558 | name, | |
1559 | in_len, | |
1560 | in_ty, | |
1561 | arg_ty: arg_tys[2], | |
1562 | out_len: element_len2 | |
1563 | } | |
dfeec247 | 1564 | ); |
94b46f34 | 1565 | |
94b46f34 XL |
1566 | // The second argument must be a simd vector with an element type that's a pointer |
1567 | // to the element type of the first argument | |
fc512014 XL |
1568 | let (_, element_ty0) = arg_tys[0].simd_size_and_type(bx.tcx()); |
1569 | let (_, element_ty1) = arg_tys[1].simd_size_and_type(bx.tcx()); | |
1570 | let (_, element_ty2) = arg_tys[2].simd_size_and_type(bx.tcx()); | |
add651ee FG |
1571 | |
1572 | require!( | |
1573 | matches!( | |
1574 | element_ty1.kind(), | |
1575 | ty::RawPtr(p) | |
1576 | if p.ty == in_elem && p.mutbl.is_mut() && p.ty.kind() == element_ty0.kind() | |
1577 | ), | |
1578 | InvalidMonomorphization::ExpectedElementType { | |
1579 | span, | |
1580 | name, | |
1581 | expected_element: element_ty1, | |
1582 | second_arg: arg_tys[1], | |
1583 | in_elem, | |
1584 | in_ty, | |
1585 | mutability: ExpectedPointerMutability::Mut, | |
94b46f34 | 1586 | } |
add651ee | 1587 | ); |
94b46f34 XL |
1588 | |
1589 | // The element type of the third argument must be a signed integer type of any width: | |
fc512014 | 1590 | match element_ty2.kind() { |
b7449926 | 1591 | ty::Int(_) => (), |
94b46f34 | 1592 | _ => { |
dfeec247 XL |
1593 | require!( |
1594 | false, | |
9c376795 FG |
1595 | InvalidMonomorphization::ThirdArgElementType { |
1596 | span, | |
1597 | name, | |
1598 | expected_element: element_ty2, | |
1599 | third_arg: arg_tys[2] | |
1600 | } | |
dfeec247 | 1601 | ); |
94b46f34 XL |
1602 | } |
1603 | } | |
1604 | ||
1605 | // Alignment of T, must be a constant integer value: | |
a1dfa0c6 XL |
1606 | let alignment_ty = bx.type_i32(); |
1607 | let alignment = bx.const_i32(bx.align_of(in_elem).bytes() as i32); | |
94b46f34 XL |
1608 | |
1609 | // Truncate the mask vector to a vector of i1s: | |
1610 | let (mask, mask_ty) = { | |
a1dfa0c6 | 1611 | let i1 = bx.type_i1(); |
60c5eb7d | 1612 | let i1xn = bx.type_vector(i1, in_len); |
94b46f34 XL |
1613 | (bx.trunc(args[2].immediate(), i1xn), i1xn) |
1614 | }; | |
1615 | ||
a1dfa0c6 | 1616 | let ret_t = bx.type_void(); |
94b46f34 XL |
1617 | |
1618 | // Type of the vector of pointers: | |
add651ee FG |
1619 | let llvm_pointer_vec_ty = llvm_vector_ty(bx, element_ty1, in_len); |
1620 | let llvm_pointer_vec_str = llvm_vector_str(bx, element_ty1, in_len); | |
94b46f34 XL |
1621 | |
1622 | // Type of the vector of elements: | |
add651ee FG |
1623 | let llvm_elem_vec_ty = llvm_vector_ty(bx, element_ty0, in_len); |
1624 | let llvm_elem_vec_str = llvm_vector_str(bx, element_ty0, in_len); | |
94b46f34 | 1625 | |
dfeec247 | 1626 | let llvm_intrinsic = |
add651ee | 1627 | format!("llvm.masked.scatter.{llvm_elem_vec_str}.{llvm_pointer_vec_str}"); |
94222f64 XL |
1628 | let fn_ty = |
1629 | bx.type_func(&[llvm_elem_vec_ty, llvm_pointer_vec_ty, alignment_ty, mask_ty], ret_t); | |
1630 | let f = bx.declare_cfn(&llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); | |
2b03887a FG |
1631 | let v = bx.call( |
1632 | fn_ty, | |
1633 | None, | |
49aad941 | 1634 | None, |
2b03887a FG |
1635 | f, |
1636 | &[args[0].immediate(), args[1].immediate(), alignment, mask], | |
1637 | None, | |
1638 | ); | |
94b46f34 XL |
1639 | return Ok(v); |
1640 | } | |
1641 | ||
0531ce1d | 1642 | macro_rules! arith_red { |
3dfed10e XL |
1643 | ($name:ident : $integer_reduce:ident, $float_reduce:ident, $ordered:expr, $op:ident, |
1644 | $identity:expr) => { | |
1645 | if name == sym::$name { | |
dfeec247 XL |
1646 | require!( |
1647 | ret_ty == in_elem, | |
9c376795 | 1648 | InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty } |
dfeec247 | 1649 | ); |
1b1a35ee | 1650 | return match in_elem.kind() { |
b7449926 | 1651 | ty::Int(_) | ty::Uint(_) => { |
0531ce1d XL |
1652 | let r = bx.$integer_reduce(args[0].immediate()); |
1653 | if $ordered { | |
1654 | // if overflow occurs, the result is the | |
1655 | // mathematical result modulo 2^n: | |
3dfed10e | 1656 | Ok(bx.$op(args[1].immediate(), r)) |
0531ce1d XL |
1657 | } else { |
1658 | Ok(bx.$integer_reduce(args[0].immediate())) | |
1659 | } | |
dfeec247 | 1660 | } |
b7449926 | 1661 | ty::Float(f) => { |
0531ce1d | 1662 | let acc = if $ordered { |
416331ca XL |
1663 | // ordered arithmetic reductions take an accumulator |
1664 | args[1].immediate() | |
0531ce1d | 1665 | } else { |
416331ca | 1666 | // unordered arithmetic reductions use the identity accumulator |
0531ce1d | 1667 | match f.bit_width() { |
3dfed10e XL |
1668 | 32 => bx.const_real(bx.type_f32(), $identity), |
1669 | 64 => bx.const_real(bx.type_f64(), $identity), | |
dfeec247 | 1670 | v => return_error!( |
9c376795 FG |
1671 | InvalidMonomorphization::UnsupportedSymbolOfSize { |
1672 | span, | |
1673 | name, | |
1674 | symbol: sym::$name, | |
1675 | in_ty, | |
1676 | in_elem, | |
1677 | size: v, | |
1678 | ret_ty | |
1679 | } | |
dfeec247 | 1680 | ), |
0531ce1d | 1681 | } |
0531ce1d XL |
1682 | }; |
1683 | Ok(bx.$float_reduce(acc, args[0].immediate())) | |
1684 | } | |
9c376795 FG |
1685 | _ => return_error!(InvalidMonomorphization::UnsupportedSymbol { |
1686 | span, | |
1687 | name, | |
1688 | symbol: sym::$name, | |
dfeec247 XL |
1689 | in_ty, |
1690 | in_elem, | |
1691 | ret_ty | |
9c376795 | 1692 | }), |
dfeec247 | 1693 | }; |
0531ce1d | 1694 | } |
dfeec247 | 1695 | }; |
0531ce1d XL |
1696 | } |
1697 | ||
3dfed10e XL |
1698 | arith_red!(simd_reduce_add_ordered: vector_reduce_add, vector_reduce_fadd, true, add, 0.0); |
1699 | arith_red!(simd_reduce_mul_ordered: vector_reduce_mul, vector_reduce_fmul, true, mul, 1.0); | |
1700 | arith_red!( | |
1701 | simd_reduce_add_unordered: vector_reduce_add, | |
1702 | vector_reduce_fadd_fast, | |
1703 | false, | |
1704 | add, | |
1705 | 0.0 | |
1706 | ); | |
1707 | arith_red!( | |
1708 | simd_reduce_mul_unordered: vector_reduce_mul, | |
1709 | vector_reduce_fmul_fast, | |
1710 | false, | |
1711 | mul, | |
1712 | 1.0 | |
1713 | ); | |
0531ce1d XL |
1714 | |
1715 | macro_rules! minmax_red { | |
3dfed10e XL |
1716 | ($name:ident: $int_red:ident, $float_red:ident) => { |
1717 | if name == sym::$name { | |
dfeec247 XL |
1718 | require!( |
1719 | ret_ty == in_elem, | |
9c376795 | 1720 | InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty } |
dfeec247 | 1721 | ); |
1b1a35ee | 1722 | return match in_elem.kind() { |
dfeec247 XL |
1723 | ty::Int(_i) => Ok(bx.$int_red(args[0].immediate(), true)), |
1724 | ty::Uint(_u) => Ok(bx.$int_red(args[0].immediate(), false)), | |
1725 | ty::Float(_f) => Ok(bx.$float_red(args[0].immediate())), | |
9c376795 FG |
1726 | _ => return_error!(InvalidMonomorphization::UnsupportedSymbol { |
1727 | span, | |
1728 | name, | |
1729 | symbol: sym::$name, | |
dfeec247 XL |
1730 | in_ty, |
1731 | in_elem, | |
1732 | ret_ty | |
9c376795 | 1733 | }), |
dfeec247 | 1734 | }; |
0531ce1d | 1735 | } |
dfeec247 | 1736 | }; |
0531ce1d XL |
1737 | } |
1738 | ||
3dfed10e XL |
1739 | minmax_red!(simd_reduce_min: vector_reduce_min, vector_reduce_fmin); |
1740 | minmax_red!(simd_reduce_max: vector_reduce_max, vector_reduce_fmax); | |
0531ce1d | 1741 | |
3dfed10e XL |
1742 | minmax_red!(simd_reduce_min_nanless: vector_reduce_min, vector_reduce_fmin_fast); |
1743 | minmax_red!(simd_reduce_max_nanless: vector_reduce_max, vector_reduce_fmax_fast); | |
0531ce1d XL |
1744 | |
1745 | macro_rules! bitwise_red { | |
3dfed10e XL |
1746 | ($name:ident : $red:ident, $boolean:expr) => { |
1747 | if name == sym::$name { | |
0531ce1d | 1748 | let input = if !$boolean { |
dfeec247 XL |
1749 | require!( |
1750 | ret_ty == in_elem, | |
9c376795 | 1751 | InvalidMonomorphization::ReturnType { span, name, in_elem, in_ty, ret_ty } |
dfeec247 | 1752 | ); |
0531ce1d XL |
1753 | args[0].immediate() |
1754 | } else { | |
1b1a35ee | 1755 | match in_elem.kind() { |
dfeec247 | 1756 | ty::Int(_) | ty::Uint(_) => {} |
9c376795 FG |
1757 | _ => return_error!(InvalidMonomorphization::UnsupportedSymbol { |
1758 | span, | |
1759 | name, | |
1760 | symbol: sym::$name, | |
dfeec247 XL |
1761 | in_ty, |
1762 | in_elem, | |
1763 | ret_ty | |
9c376795 | 1764 | }), |
0531ce1d XL |
1765 | } |
1766 | ||
1767 | // boolean reductions operate on vectors of i1s: | |
a1dfa0c6 XL |
1768 | let i1 = bx.type_i1(); |
1769 | let i1xn = bx.type_vector(i1, in_len as u64); | |
0531ce1d XL |
1770 | bx.trunc(args[0].immediate(), i1xn) |
1771 | }; | |
1b1a35ee | 1772 | return match in_elem.kind() { |
b7449926 | 1773 | ty::Int(_) | ty::Uint(_) => { |
0531ce1d | 1774 | let r = bx.$red(input); |
dfeec247 XL |
1775 | Ok(if !$boolean { r } else { bx.zext(r, bx.type_bool()) }) |
1776 | } | |
9c376795 FG |
1777 | _ => return_error!(InvalidMonomorphization::UnsupportedSymbol { |
1778 | span, | |
1779 | name, | |
1780 | symbol: sym::$name, | |
dfeec247 XL |
1781 | in_ty, |
1782 | in_elem, | |
1783 | ret_ty | |
9c376795 | 1784 | }), |
dfeec247 | 1785 | }; |
0531ce1d | 1786 | } |
dfeec247 | 1787 | }; |
0531ce1d XL |
1788 | } |
1789 | ||
3dfed10e XL |
1790 | bitwise_red!(simd_reduce_and: vector_reduce_and, false); |
1791 | bitwise_red!(simd_reduce_or: vector_reduce_or, false); | |
1792 | bitwise_red!(simd_reduce_xor: vector_reduce_xor, false); | |
1793 | bitwise_red!(simd_reduce_all: vector_reduce_and, true); | |
1794 | bitwise_red!(simd_reduce_any: vector_reduce_or, true); | |
0531ce1d | 1795 | |
f2b60f7d | 1796 | if name == sym::simd_cast_ptr { |
9c376795 | 1797 | require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty }); |
f2b60f7d FG |
1798 | let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx()); |
1799 | require!( | |
1800 | in_len == out_len, | |
9c376795 FG |
1801 | InvalidMonomorphization::ReturnLengthInputType { |
1802 | span, | |
1803 | name, | |
1804 | in_len, | |
1805 | in_ty, | |
1806 | ret_ty, | |
1807 | out_len | |
1808 | } | |
f2b60f7d FG |
1809 | ); |
1810 | ||
1811 | match in_elem.kind() { | |
1812 | ty::RawPtr(p) => { | |
1813 | let (metadata, check_sized) = p.ty.ptr_metadata_ty(bx.tcx, |ty| { | |
1814 | bx.tcx.normalize_erasing_regions(ty::ParamEnv::reveal_all(), ty) | |
1815 | }); | |
1816 | assert!(!check_sized); // we are in codegen, so we shouldn't see these types | |
9c376795 FG |
1817 | require!( |
1818 | metadata.is_unit(), | |
1819 | InvalidMonomorphization::CastFatPointer { span, name, ty: in_elem } | |
1820 | ); | |
1821 | } | |
1822 | _ => { | |
1823 | return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: in_elem }) | |
f2b60f7d | 1824 | } |
f2b60f7d FG |
1825 | } |
1826 | match out_elem.kind() { | |
1827 | ty::RawPtr(p) => { | |
1828 | let (metadata, check_sized) = p.ty.ptr_metadata_ty(bx.tcx, |ty| { | |
1829 | bx.tcx.normalize_erasing_regions(ty::ParamEnv::reveal_all(), ty) | |
1830 | }); | |
1831 | assert!(!check_sized); // we are in codegen, so we shouldn't see these types | |
9c376795 FG |
1832 | require!( |
1833 | metadata.is_unit(), | |
1834 | InvalidMonomorphization::CastFatPointer { span, name, ty: out_elem } | |
1835 | ); | |
1836 | } | |
1837 | _ => { | |
1838 | return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: out_elem }) | |
f2b60f7d | 1839 | } |
f2b60f7d FG |
1840 | } |
1841 | ||
add651ee | 1842 | return Ok(args[0].immediate()); |
f2b60f7d FG |
1843 | } |
1844 | ||
1845 | if name == sym::simd_expose_addr { | |
9c376795 | 1846 | require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty }); |
f2b60f7d FG |
1847 | let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx()); |
1848 | require!( | |
1849 | in_len == out_len, | |
9c376795 FG |
1850 | InvalidMonomorphization::ReturnLengthInputType { |
1851 | span, | |
1852 | name, | |
1853 | in_len, | |
1854 | in_ty, | |
1855 | ret_ty, | |
1856 | out_len | |
1857 | } | |
f2b60f7d FG |
1858 | ); |
1859 | ||
1860 | match in_elem.kind() { | |
1861 | ty::RawPtr(_) => {} | |
9c376795 FG |
1862 | _ => { |
1863 | return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: in_elem }) | |
1864 | } | |
f2b60f7d FG |
1865 | } |
1866 | match out_elem.kind() { | |
1867 | ty::Uint(ty::UintTy::Usize) => {} | |
9c376795 | 1868 | _ => return_error!(InvalidMonomorphization::ExpectedUsize { span, name, ty: out_elem }), |
f2b60f7d FG |
1869 | } |
1870 | ||
1871 | return Ok(bx.ptrtoint(args[0].immediate(), llret_ty)); | |
1872 | } | |
1873 | ||
1874 | if name == sym::simd_from_exposed_addr { | |
9c376795 | 1875 | require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty }); |
f2b60f7d FG |
1876 | let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx()); |
1877 | require!( | |
1878 | in_len == out_len, | |
9c376795 FG |
1879 | InvalidMonomorphization::ReturnLengthInputType { |
1880 | span, | |
1881 | name, | |
1882 | in_len, | |
1883 | in_ty, | |
1884 | ret_ty, | |
1885 | out_len | |
1886 | } | |
f2b60f7d FG |
1887 | ); |
1888 | ||
1889 | match in_elem.kind() { | |
1890 | ty::Uint(ty::UintTy::Usize) => {} | |
9c376795 | 1891 | _ => return_error!(InvalidMonomorphization::ExpectedUsize { span, name, ty: in_elem }), |
f2b60f7d FG |
1892 | } |
1893 | match out_elem.kind() { | |
1894 | ty::RawPtr(_) => {} | |
9c376795 FG |
1895 | _ => { |
1896 | return_error!(InvalidMonomorphization::ExpectedPointer { span, name, ty: out_elem }) | |
1897 | } | |
f2b60f7d FG |
1898 | } |
1899 | ||
1900 | return Ok(bx.inttoptr(args[0].immediate(), llret_ty)); | |
1901 | } | |
1902 | ||
5099ac24 | 1903 | if name == sym::simd_cast || name == sym::simd_as { |
9c376795 | 1904 | require_simd!(ret_ty, InvalidMonomorphization::SimdReturn { span, name, ty: ret_ty }); |
fc512014 | 1905 | let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx()); |
dfeec247 XL |
1906 | require!( |
1907 | in_len == out_len, | |
9c376795 FG |
1908 | InvalidMonomorphization::ReturnLengthInputType { |
1909 | span, | |
1910 | name, | |
1911 | in_len, | |
1912 | in_ty, | |
1913 | ret_ty, | |
1914 | out_len | |
1915 | } | |
dfeec247 | 1916 | ); |
e9174d1e | 1917 | // casting cares about nominal type, not just structural type |
dfeec247 XL |
1918 | if in_elem == out_elem { |
1919 | return Ok(args[0].immediate()); | |
1920 | } | |
e9174d1e | 1921 | |
dfeec247 XL |
1922 | enum Style { |
1923 | Float, | |
1924 | Int(/* is signed? */ bool), | |
1925 | Unsupported, | |
1926 | } | |
e9174d1e | 1927 | |
1b1a35ee | 1928 | let (in_style, in_width) = match in_elem.kind() { |
e9174d1e SL |
1929 | // vectors of pointer-sized integers should've been |
1930 | // disallowed before here, so this unwrap is safe. | |
5099ac24 FG |
1931 | ty::Int(i) => ( |
1932 | Style::Int(true), | |
1933 | i.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(), | |
1934 | ), | |
1935 | ty::Uint(u) => ( | |
1936 | Style::Int(false), | |
1937 | u.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(), | |
1938 | ), | |
b7449926 | 1939 | ty::Float(f) => (Style::Float, f.bit_width()), |
dfeec247 | 1940 | _ => (Style::Unsupported, 0), |
e9174d1e | 1941 | }; |
1b1a35ee | 1942 | let (out_style, out_width) = match out_elem.kind() { |
5099ac24 FG |
1943 | ty::Int(i) => ( |
1944 | Style::Int(true), | |
1945 | i.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(), | |
1946 | ), | |
1947 | ty::Uint(u) => ( | |
1948 | Style::Int(false), | |
1949 | u.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(), | |
1950 | ), | |
b7449926 | 1951 | ty::Float(f) => (Style::Float, f.bit_width()), |
dfeec247 | 1952 | _ => (Style::Unsupported, 0), |
e9174d1e SL |
1953 | }; |
1954 | ||
1955 | match (in_style, out_style) { | |
1956 | (Style::Int(in_is_signed), Style::Int(_)) => { | |
ff7c6d11 | 1957 | return Ok(match in_width.cmp(&out_width) { |
2c00a5a8 | 1958 | Ordering::Greater => bx.trunc(args[0].immediate(), llret_ty), |
ff7c6d11 | 1959 | Ordering::Equal => args[0].immediate(), |
dfeec247 XL |
1960 | Ordering::Less => { |
1961 | if in_is_signed { | |
1962 | bx.sext(args[0].immediate(), llret_ty) | |
1963 | } else { | |
1964 | bx.zext(args[0].immediate(), llret_ty) | |
1965 | } | |
e9174d1e | 1966 | } |
dfeec247 | 1967 | }); |
e9174d1e SL |
1968 | } |
1969 | (Style::Int(in_is_signed), Style::Float) => { | |
ff7c6d11 | 1970 | return Ok(if in_is_signed { |
2c00a5a8 | 1971 | bx.sitofp(args[0].immediate(), llret_ty) |
e9174d1e | 1972 | } else { |
2c00a5a8 | 1973 | bx.uitofp(args[0].immediate(), llret_ty) |
dfeec247 | 1974 | }); |
e9174d1e SL |
1975 | } |
1976 | (Style::Float, Style::Int(out_is_signed)) => { | |
5099ac24 FG |
1977 | return Ok(match (out_is_signed, name == sym::simd_as) { |
1978 | (false, false) => bx.fptoui(args[0].immediate(), llret_ty), | |
1979 | (true, false) => bx.fptosi(args[0].immediate(), llret_ty), | |
1980 | (_, true) => bx.cast_float_to_int(out_is_signed, args[0].immediate(), llret_ty), | |
dfeec247 | 1981 | }); |
e9174d1e SL |
1982 | } |
1983 | (Style::Float, Style::Float) => { | |
ff7c6d11 | 1984 | return Ok(match in_width.cmp(&out_width) { |
2c00a5a8 | 1985 | Ordering::Greater => bx.fptrunc(args[0].immediate(), llret_ty), |
ff7c6d11 | 1986 | Ordering::Equal => args[0].immediate(), |
dfeec247 XL |
1987 | Ordering::Less => bx.fpext(args[0].immediate(), llret_ty), |
1988 | }); | |
e9174d1e | 1989 | } |
dfeec247 | 1990 | _ => { /* Unsupported. Fallthrough. */ } |
e9174d1e | 1991 | } |
dfeec247 XL |
1992 | require!( |
1993 | false, | |
9c376795 FG |
1994 | InvalidMonomorphization::UnsupportedCast { |
1995 | span, | |
1996 | name, | |
1997 | in_ty, | |
1998 | in_elem, | |
1999 | ret_ty, | |
2000 | out_elem | |
2001 | } | |
dfeec247 | 2002 | ); |
e9174d1e | 2003 | } |
6a06907d | 2004 | macro_rules! arith_binary { |
32a655c1 | 2005 | ($($name: ident: $($($p: ident),* => $call: ident),*;)*) => { |
3dfed10e | 2006 | $(if name == sym::$name { |
1b1a35ee | 2007 | match in_elem.kind() { |
ff7c6d11 | 2008 | $($(ty::$p(_))|* => { |
2c00a5a8 | 2009 | return Ok(bx.$call(args[0].immediate(), args[1].immediate())) |
ff7c6d11 XL |
2010 | })* |
2011 | _ => {}, | |
2012 | } | |
9c376795 FG |
2013 | require!( |
2014 | false, | |
2015 | InvalidMonomorphization::UnsupportedOperation { span, name, in_ty, in_elem } | |
2016 | ); | |
ff7c6d11 | 2017 | })* |
e9174d1e SL |
2018 | } |
2019 | } | |
6a06907d | 2020 | arith_binary! { |
b7449926 XL |
2021 | simd_add: Uint, Int => add, Float => fadd; |
2022 | simd_sub: Uint, Int => sub, Float => fsub; | |
2023 | simd_mul: Uint, Int => mul, Float => fmul; | |
2024 | simd_div: Uint => udiv, Int => sdiv, Float => fdiv; | |
2025 | simd_rem: Uint => urem, Int => srem, Float => frem; | |
2026 | simd_shl: Uint, Int => shl; | |
2027 | simd_shr: Uint => lshr, Int => ashr; | |
2028 | simd_and: Uint, Int => and; | |
2029 | simd_or: Uint, Int => or; | |
2030 | simd_xor: Uint, Int => xor; | |
2031 | simd_fmax: Float => maxnum; | |
2032 | simd_fmin: Float => minnum; | |
9fa01778 | 2033 | |
e9174d1e | 2034 | } |
6a06907d XL |
2035 | macro_rules! arith_unary { |
2036 | ($($name: ident: $($($p: ident),* => $call: ident),*;)*) => { | |
2037 | $(if name == sym::$name { | |
2038 | match in_elem.kind() { | |
2039 | $($(ty::$p(_))|* => { | |
2040 | return Ok(bx.$call(args[0].immediate())) | |
2041 | })* | |
2042 | _ => {}, | |
2043 | } | |
9c376795 FG |
2044 | require!( |
2045 | false, | |
2046 | InvalidMonomorphization::UnsupportedOperation { span, name, in_ty, in_elem } | |
2047 | ); | |
6a06907d XL |
2048 | })* |
2049 | } | |
2050 | } | |
2051 | arith_unary! { | |
2052 | simd_neg: Int => neg, Float => fneg; | |
2053 | } | |
9fa01778 | 2054 | |
add651ee FG |
2055 | // Unary integer intrinsics |
2056 | if matches!(name, sym::simd_bswap | sym::simd_bitreverse | sym::simd_ctlz | sym::simd_cttz) { | |
2057 | let vec_ty = bx.cx.type_vector( | |
2058 | match *in_elem.kind() { | |
2059 | ty::Int(i) => bx.cx.type_int_from_ty(i), | |
2060 | ty::Uint(i) => bx.cx.type_uint_from_ty(i), | |
2061 | _ => return_error!(InvalidMonomorphization::UnsupportedOperation { | |
2062 | span, | |
2063 | name, | |
2064 | in_ty, | |
2065 | in_elem | |
2066 | }), | |
2067 | }, | |
2068 | in_len as u64, | |
2069 | ); | |
2070 | let intrinsic_name = match name { | |
2071 | sym::simd_bswap => "bswap", | |
2072 | sym::simd_bitreverse => "bitreverse", | |
2073 | sym::simd_ctlz => "ctlz", | |
2074 | sym::simd_cttz => "cttz", | |
2075 | _ => unreachable!(), | |
2076 | }; | |
2077 | let int_size = in_elem.int_size_and_signed(bx.tcx()).0.bits(); | |
2078 | let llvm_intrinsic = &format!("llvm.{}.v{}i{}", intrinsic_name, in_len, int_size,); | |
2079 | ||
2080 | return if name == sym::simd_bswap && int_size == 8 { | |
2081 | // byte swap is no-op for i8/u8 | |
2082 | Ok(args[0].immediate()) | |
2083 | } else if matches!(name, sym::simd_ctlz | sym::simd_cttz) { | |
2084 | let fn_ty = bx.type_func(&[vec_ty, bx.type_i1()], vec_ty); | |
2085 | let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); | |
2086 | Ok(bx.call( | |
2087 | fn_ty, | |
2088 | None, | |
2089 | None, | |
2090 | f, | |
2091 | &[args[0].immediate(), bx.const_int(bx.type_i1(), 0)], | |
2092 | None, | |
2093 | )) | |
2094 | } else { | |
2095 | let fn_ty = bx.type_func(&[vec_ty], vec_ty); | |
2096 | let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); | |
2097 | Ok(bx.call(fn_ty, None, None, f, &[args[0].immediate()], None)) | |
2098 | }; | |
2099 | } | |
2100 | ||
04454e1e FG |
2101 | if name == sym::simd_arith_offset { |
2102 | // This also checks that the first operand is a ptr type. | |
2103 | let pointee = in_elem.builtin_deref(true).unwrap_or_else(|| { | |
2104 | span_bug!(span, "must be called with a vector of pointer types as first argument") | |
2105 | }); | |
2106 | let layout = bx.layout_of(pointee.ty); | |
2107 | let ptrs = args[0].immediate(); | |
2108 | // The second argument must be a ptr-sized integer. | |
2109 | // (We don't care about the signedness, this is wrapping anyway.) | |
2110 | let (_offsets_len, offsets_elem) = arg_tys[1].simd_size_and_type(bx.tcx()); | |
2111 | if !matches!(offsets_elem.kind(), ty::Int(ty::IntTy::Isize) | ty::Uint(ty::UintTy::Usize)) { | |
2112 | span_bug!( | |
2113 | span, | |
2114 | "must be called with a vector of pointer-sized integers as second argument" | |
2115 | ); | |
2116 | } | |
2117 | let offsets = args[1].immediate(); | |
2118 | ||
2119 | return Ok(bx.gep(bx.backend_type(layout), ptrs, &[offsets])); | |
2120 | } | |
2121 | ||
3dfed10e | 2122 | if name == sym::simd_saturating_add || name == sym::simd_saturating_sub { |
9fa01778 XL |
2123 | let lhs = args[0].immediate(); |
2124 | let rhs = args[1].immediate(); | |
3dfed10e | 2125 | let is_add = name == sym::simd_saturating_add; |
9fa01778 | 2126 | let ptr_bits = bx.tcx().data_layout.pointer_size.bits() as _; |
1b1a35ee | 2127 | let (signed, elem_width, elem_ty) = match *in_elem.kind() { |
dfeec247 XL |
2128 | ty::Int(i) => (true, i.bit_width().unwrap_or(ptr_bits), bx.cx.type_int_from_ty(i)), |
2129 | ty::Uint(i) => (false, i.bit_width().unwrap_or(ptr_bits), bx.cx.type_uint_from_ty(i)), | |
9fa01778 | 2130 | _ => { |
9c376795 FG |
2131 | return_error!(InvalidMonomorphization::ExpectedVectorElementType { |
2132 | span, | |
2133 | name, | |
2134 | expected_element: arg_tys[0].simd_size_and_type(bx.tcx()).1, | |
2135 | vector_type: arg_tys[0] | |
2136 | }); | |
9fa01778 XL |
2137 | } |
2138 | }; | |
2139 | let llvm_intrinsic = &format!( | |
2140 | "llvm.{}{}.sat.v{}i{}", | |
2141 | if signed { 's' } else { 'u' }, | |
2142 | if is_add { "add" } else { "sub" }, | |
dfeec247 XL |
2143 | in_len, |
2144 | elem_width | |
9fa01778 XL |
2145 | ); |
2146 | let vec_ty = bx.cx.type_vector(elem_ty, in_len as u64); | |
2147 | ||
94222f64 | 2148 | let fn_ty = bx.type_func(&[vec_ty, vec_ty], vec_ty); |
c295e0f8 | 2149 | let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); |
49aad941 | 2150 | let v = bx.call(fn_ty, None, None, f, &[lhs, rhs], None); |
9fa01778 XL |
2151 | return Ok(v); |
2152 | } | |
2153 | ||
54a0048b | 2154 | span_bug!(span, "unknown SIMD intrinsic"); |
e9174d1e | 2155 | } |
92a42be0 | 2156 | |
ff7c6d11 | 2157 | // Returns the width of an int Ty, and if it's signed or not |
92a42be0 | 2158 | // Returns None if the type is not an integer |
32a655c1 SL |
2159 | // FIXME: there’s multiple of this functions, investigate using some of the already existing |
2160 | // stuffs. | |
9fa01778 | 2161 | fn int_type_width_signed(ty: Ty<'_>, cx: &CodegenCx<'_, '_>) -> Option<(u64, bool)> { |
1b1a35ee XL |
2162 | match ty.kind() { |
2163 | ty::Int(t) => { | |
29967ef6 | 2164 | Some((t.bit_width().unwrap_or(u64::from(cx.tcx.sess.target.pointer_width)), true)) |
1b1a35ee XL |
2165 | } |
2166 | ty::Uint(t) => { | |
29967ef6 | 2167 | Some((t.bit_width().unwrap_or(u64::from(cx.tcx.sess.target.pointer_width)), false)) |
1b1a35ee | 2168 | } |
54a0048b SL |
2169 | _ => None, |
2170 | } | |
2171 | } |