]>
Commit | Line | Data |
---|---|---|
60c5eb7d | 1 | use crate::abi::{Abi, FnAbi, LlvmType, PassMode}; |
dfeec247 | 2 | use crate::builder::Builder; |
9fa01778 | 3 | use crate::context::CodegenCx; |
dfeec247 | 4 | use crate::llvm; |
9fa01778 XL |
5 | use crate::type_::Type; |
6 | use crate::type_of::LayoutLlvmExt; | |
9fa01778 | 7 | use crate::va_arg::emit_va_arg; |
dfeec247 | 8 | use crate::value::Value; |
ba9703b0 | 9 | |
f035d41b XL |
10 | use log::debug; |
11 | ||
74b04a01 | 12 | use rustc_ast::ast; |
dfeec247 | 13 | use rustc_codegen_ssa::base::{compare_simd_types, to_immediate, wants_msvc_seh}; |
ba9703b0 | 14 | use rustc_codegen_ssa::common::span_invalid_monomorphization_error; |
f035d41b XL |
15 | use rustc_codegen_ssa::common::{IntPredicate, TypeKind}; |
16 | use rustc_codegen_ssa::coverageinfo::CounterOp; | |
dfeec247 XL |
17 | use rustc_codegen_ssa::glue; |
18 | use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue}; | |
19 | use rustc_codegen_ssa::mir::place::PlaceRef; | |
ba9703b0 | 20 | use rustc_codegen_ssa::traits::*; |
dfeec247 XL |
21 | use rustc_codegen_ssa::MemFlags; |
22 | use rustc_hir as hir; | |
f035d41b XL |
23 | use rustc_middle::mir::coverage; |
24 | use rustc_middle::mir::Operand; | |
ba9703b0 XL |
25 | use rustc_middle::ty::layout::{FnAbiExt, HasTyCtxt}; |
26 | use rustc_middle::ty::{self, Ty}; | |
27 | use rustc_middle::{bug, span_bug}; | |
dfeec247 | 28 | use rustc_span::Span; |
ba9703b0 | 29 | use rustc_target::abi::{self, HasDataLayout, LayoutOf, Primitive}; |
f9f354fc | 30 | use rustc_target::spec::PanicStrategy; |
32a655c1 | 31 | |
e9174d1e | 32 | use std::cmp::Ordering; |
ba9703b0 | 33 | use std::iter; |
e9174d1e | 34 | |
b7449926 | 35 | fn get_simple_intrinsic(cx: &CodegenCx<'ll, '_>, name: &str) -> Option<&'ll Value> { |
54a0048b | 36 | let llvm_name = match name { |
1a4d82fc JJ |
37 | "sqrtf32" => "llvm.sqrt.f32", |
38 | "sqrtf64" => "llvm.sqrt.f64", | |
39 | "powif32" => "llvm.powi.f32", | |
40 | "powif64" => "llvm.powi.f64", | |
41 | "sinf32" => "llvm.sin.f32", | |
42 | "sinf64" => "llvm.sin.f64", | |
43 | "cosf32" => "llvm.cos.f32", | |
44 | "cosf64" => "llvm.cos.f64", | |
45 | "powf32" => "llvm.pow.f32", | |
46 | "powf64" => "llvm.pow.f64", | |
47 | "expf32" => "llvm.exp.f32", | |
48 | "expf64" => "llvm.exp.f64", | |
49 | "exp2f32" => "llvm.exp2.f32", | |
50 | "exp2f64" => "llvm.exp2.f64", | |
51 | "logf32" => "llvm.log.f32", | |
52 | "logf64" => "llvm.log.f64", | |
53 | "log10f32" => "llvm.log10.f32", | |
54 | "log10f64" => "llvm.log10.f64", | |
55 | "log2f32" => "llvm.log2.f32", | |
56 | "log2f64" => "llvm.log2.f64", | |
57 | "fmaf32" => "llvm.fma.f32", | |
58 | "fmaf64" => "llvm.fma.f64", | |
59 | "fabsf32" => "llvm.fabs.f32", | |
60 | "fabsf64" => "llvm.fabs.f64", | |
dc9dc135 XL |
61 | "minnumf32" => "llvm.minnum.f32", |
62 | "minnumf64" => "llvm.minnum.f64", | |
63 | "maxnumf32" => "llvm.maxnum.f32", | |
64 | "maxnumf64" => "llvm.maxnum.f64", | |
1a4d82fc JJ |
65 | "copysignf32" => "llvm.copysign.f32", |
66 | "copysignf64" => "llvm.copysign.f64", | |
67 | "floorf32" => "llvm.floor.f32", | |
68 | "floorf64" => "llvm.floor.f64", | |
69 | "ceilf32" => "llvm.ceil.f32", | |
70 | "ceilf64" => "llvm.ceil.f64", | |
71 | "truncf32" => "llvm.trunc.f32", | |
72 | "truncf64" => "llvm.trunc.f64", | |
73 | "rintf32" => "llvm.rint.f32", | |
74 | "rintf64" => "llvm.rint.f64", | |
75 | "nearbyintf32" => "llvm.nearbyint.f32", | |
76 | "nearbyintf64" => "llvm.nearbyint.f64", | |
77 | "roundf32" => "llvm.round.f32", | |
78 | "roundf64" => "llvm.round.f64", | |
1a4d82fc | 79 | "assume" => "llvm.assume", |
32a655c1 | 80 | "abort" => "llvm.trap", |
dfeec247 | 81 | _ => return None, |
1a4d82fc | 82 | }; |
2c00a5a8 | 83 | Some(cx.get_intrinsic(&llvm_name)) |
1a4d82fc JJ |
84 | } |
85 | ||
a1dfa0c6 | 86 | impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> { |
f035d41b XL |
87 | fn is_codegen_intrinsic( |
88 | &mut self, | |
89 | intrinsic: &str, | |
90 | args: &Vec<Operand<'tcx>>, | |
91 | caller_instance: ty::Instance<'tcx>, | |
92 | ) -> bool { | |
93 | match intrinsic { | |
94 | "count_code_region" => { | |
95 | use coverage::count_code_region_args::*; | |
96 | self.add_counter_region( | |
97 | caller_instance, | |
98 | op_to_u32(&args[COUNTER_INDEX]), | |
99 | op_to_u32(&args[START_BYTE_POS]), | |
100 | op_to_u32(&args[END_BYTE_POS]), | |
101 | ); | |
102 | true // Also inject the counter increment in the backend | |
103 | } | |
104 | "coverage_counter_add" | "coverage_counter_subtract" => { | |
105 | use coverage::coverage_counter_expression_args::*; | |
106 | self.add_counter_expression_region( | |
107 | caller_instance, | |
108 | op_to_u32(&args[COUNTER_EXPRESSION_INDEX]), | |
109 | op_to_u32(&args[LEFT_INDEX]), | |
110 | if intrinsic == "coverage_counter_add" { | |
111 | CounterOp::Add | |
112 | } else { | |
113 | CounterOp::Subtract | |
114 | }, | |
115 | op_to_u32(&args[RIGHT_INDEX]), | |
116 | op_to_u32(&args[START_BYTE_POS]), | |
117 | op_to_u32(&args[END_BYTE_POS]), | |
118 | ); | |
119 | false // Does not inject backend code | |
120 | } | |
121 | "coverage_unreachable" => { | |
122 | use coverage::coverage_unreachable_args::*; | |
123 | self.add_unreachable_region( | |
124 | caller_instance, | |
125 | op_to_u32(&args[START_BYTE_POS]), | |
126 | op_to_u32(&args[END_BYTE_POS]), | |
127 | ); | |
128 | false // Does not inject backend code | |
129 | } | |
130 | _ => true, // Unhandled intrinsics should be passed to `codegen_intrinsic_call()` | |
131 | } | |
132 | } | |
133 | ||
a1dfa0c6 XL |
134 | fn codegen_intrinsic_call( |
135 | &mut self, | |
e1599b0c | 136 | instance: ty::Instance<'tcx>, |
60c5eb7d | 137 | fn_abi: &FnAbi<'tcx, Ty<'tcx>>, |
a1dfa0c6 XL |
138 | args: &[OperandRef<'tcx, &'ll Value>], |
139 | llresult: &'ll Value, | |
140 | span: Span, | |
f035d41b | 141 | caller_instance: ty::Instance<'tcx>, |
a1dfa0c6 XL |
142 | ) { |
143 | let tcx = self.tcx; | |
dfeec247 | 144 | let callee_ty = instance.monomorphic_ty(tcx); |
c34b1796 | 145 | |
e74abb32 | 146 | let (def_id, substs) = match callee_ty.kind { |
a1dfa0c6 | 147 | ty::FnDef(def_id, substs) => (def_id, substs), |
dfeec247 | 148 | _ => bug!("expected fn item type, found {}", callee_ty), |
a1dfa0c6 | 149 | }; |
9e0c209e | 150 | |
a1dfa0c6 XL |
151 | let sig = callee_ty.fn_sig(tcx); |
152 | let sig = tcx.normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), &sig); | |
153 | let arg_tys = sig.inputs(); | |
154 | let ret_ty = sig.output(); | |
155 | let name = &*tcx.item_name(def_id).as_str(); | |
156 | ||
157 | let llret_ty = self.layout_of(ret_ty).llvm_type(self); | |
60c5eb7d | 158 | let result = PlaceRef::new_sized(llresult, fn_abi.ret.layout); |
a1dfa0c6 XL |
159 | |
160 | let simple = get_simple_intrinsic(self, name); | |
161 | let llval = match name { | |
dfeec247 XL |
162 | _ if simple.is_some() => self.call( |
163 | simple.unwrap(), | |
164 | &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(), | |
165 | None, | |
166 | ), | |
a1dfa0c6 XL |
167 | "unreachable" => { |
168 | return; | |
dfeec247 | 169 | } |
a1dfa0c6 XL |
170 | "likely" => { |
171 | let expect = self.get_intrinsic(&("llvm.expect.i1")); | |
172 | self.call(expect, &[args[0].immediate(), self.const_bool(true)], None) | |
d9579d0f | 173 | } |
a1dfa0c6 XL |
174 | "unlikely" => { |
175 | let expect = self.get_intrinsic(&("llvm.expect.i1")); | |
176 | self.call(expect, &[args[0].immediate(), self.const_bool(false)], None) | |
1a4d82fc | 177 | } |
a1dfa0c6 | 178 | "try" => { |
dfeec247 XL |
179 | try_intrinsic( |
180 | self, | |
181 | args[0].immediate(), | |
182 | args[1].immediate(), | |
183 | args[2].immediate(), | |
184 | llresult, | |
185 | ); | |
a1dfa0c6 | 186 | return; |
54a0048b | 187 | } |
a1dfa0c6 XL |
188 | "breakpoint" => { |
189 | let llfn = self.get_intrinsic(&("llvm.debugtrap")); | |
190 | self.call(llfn, &[], None) | |
d9579d0f | 191 | } |
f035d41b XL |
192 | "count_code_region" => { |
193 | // FIXME(richkadel): The current implementation assumes the MIR for the given | |
194 | // caller_instance represents a single function. Validate and/or correct if inlining | |
195 | // and/or monomorphization invalidates these assumptions. | |
196 | let coverageinfo = tcx.coverageinfo(caller_instance.def_id()); | |
197 | let mangled_fn = tcx.symbol_name(caller_instance); | |
198 | let (mangled_fn_name, _len_val) = self.const_str(mangled_fn.name); | |
199 | let hash = self.const_u64(coverageinfo.hash); | |
200 | let num_counters = self.const_u32(coverageinfo.num_counters); | |
201 | use coverage::count_code_region_args::*; | |
202 | let index = args[COUNTER_INDEX].immediate(); | |
203 | debug!( | |
204 | "count_code_region to LLVM intrinsic instrprof.increment(fn_name={}, hash={:?}, num_counters={:?}, index={:?})", | |
205 | mangled_fn.name, hash, num_counters, index, | |
206 | ); | |
207 | self.instrprof_increment(mangled_fn_name, hash, num_counters, index) | |
208 | } | |
dfeec247 XL |
209 | "va_start" => self.va_start(args[0].immediate()), |
210 | "va_end" => self.va_end(args[0].immediate()), | |
a1dfa0c6 | 211 | "va_copy" => { |
a1dfa0c6 | 212 | let intrinsic = self.cx().get_intrinsic(&("llvm.va_copy")); |
dc9dc135 | 213 | self.call(intrinsic, &[args[0].immediate(), args[1].immediate()], None) |
a1dfa0c6 XL |
214 | } |
215 | "va_arg" => { | |
60c5eb7d | 216 | match fn_abi.ret.layout.abi { |
ba9703b0 | 217 | abi::Abi::Scalar(ref scalar) => { |
a1dfa0c6 XL |
218 | match scalar.value { |
219 | Primitive::Int(..) => { | |
220 | if self.cx().size_of(ret_ty).bytes() < 4 { | |
60c5eb7d | 221 | // `va_arg` should not be called on a integer type |
a1dfa0c6 XL |
222 | // less than 4 bytes in length. If it is, promote |
223 | // the integer to a `i32` and truncate the result | |
224 | // back to the smaller type. | |
dfeec247 | 225 | let promoted_result = emit_va_arg(self, args[0], tcx.types.i32); |
a1dfa0c6 XL |
226 | self.trunc(promoted_result, llret_ty) |
227 | } else { | |
228 | emit_va_arg(self, args[0], ret_ty) | |
229 | } | |
230 | } | |
dfeec247 | 231 | Primitive::F64 | Primitive::Pointer => { |
a1dfa0c6 XL |
232 | emit_va_arg(self, args[0], ret_ty) |
233 | } | |
234 | // `va_arg` should never be used with the return type f32. | |
dfeec247 | 235 | Primitive::F32 => bug!("the va_arg intrinsic does not work with `f32`"), |
92a42be0 | 236 | } |
a1dfa0c6 | 237 | } |
dfeec247 | 238 | _ => bug!("the va_arg intrinsic does not work with non-scalar types"), |
54a0048b SL |
239 | } |
240 | } | |
a1dfa0c6 XL |
241 | "size_of_val" => { |
242 | let tp_ty = substs.type_at(0); | |
243 | if let OperandValue::Pair(_, meta) = args[0].val { | |
9fa01778 | 244 | let (llsize, _) = glue::size_and_align_of_dst(self, tp_ty, Some(meta)); |
a1dfa0c6 XL |
245 | llsize |
246 | } else { | |
247 | self.const_usize(self.size_of(tp_ty).bytes()) | |
92a42be0 SL |
248 | } |
249 | } | |
a1dfa0c6 XL |
250 | "min_align_of_val" => { |
251 | let tp_ty = substs.type_at(0); | |
252 | if let OperandValue::Pair(_, meta) = args[0].val { | |
9fa01778 | 253 | let (_, llalign) = glue::size_and_align_of_dst(self, tp_ty, Some(meta)); |
a1dfa0c6 XL |
254 | llalign |
255 | } else { | |
256 | self.const_usize(self.align_of(tp_ty).bytes()) | |
257 | } | |
258 | } | |
dfeec247 | 259 | "size_of" | "pref_align_of" | "min_align_of" | "needs_drop" | "type_id" |
f035d41b | 260 | | "type_name" | "variant_count" => { |
f9f354fc | 261 | let value = self |
dfeec247 XL |
262 | .tcx |
263 | .const_eval_instance(ty::ParamEnv::reveal_all(), instance, None) | |
264 | .unwrap(); | |
f9f354fc | 265 | OperandRef::from_const(self, value, ret_ty).immediate_or_packed_pair(self) |
a1dfa0c6 | 266 | } |
ba9703b0 XL |
267 | // Effectively no-op |
268 | "forget" => { | |
a1dfa0c6 XL |
269 | return; |
270 | } | |
a1dfa0c6 XL |
271 | "offset" => { |
272 | let ptr = args[0].immediate(); | |
273 | let offset = args[1].immediate(); | |
274 | self.inbounds_gep(ptr, &[offset]) | |
275 | } | |
276 | "arith_offset" => { | |
277 | let ptr = args[0].immediate(); | |
278 | let offset = args[1].immediate(); | |
279 | self.gep(ptr, &[offset]) | |
ff7c6d11 | 280 | } |
1a4d82fc | 281 | |
a1dfa0c6 | 282 | "copy_nonoverlapping" => { |
dfeec247 XL |
283 | copy_intrinsic( |
284 | self, | |
285 | false, | |
286 | false, | |
287 | substs.type_at(0), | |
288 | args[1].immediate(), | |
289 | args[0].immediate(), | |
290 | args[2].immediate(), | |
291 | ); | |
a1dfa0c6 XL |
292 | return; |
293 | } | |
294 | "copy" => { | |
dfeec247 XL |
295 | copy_intrinsic( |
296 | self, | |
297 | true, | |
298 | false, | |
299 | substs.type_at(0), | |
300 | args[1].immediate(), | |
301 | args[0].immediate(), | |
302 | args[2].immediate(), | |
303 | ); | |
a1dfa0c6 XL |
304 | return; |
305 | } | |
306 | "write_bytes" => { | |
dfeec247 XL |
307 | memset_intrinsic( |
308 | self, | |
309 | false, | |
310 | substs.type_at(0), | |
311 | args[0].immediate(), | |
312 | args[1].immediate(), | |
313 | args[2].immediate(), | |
314 | ); | |
a1dfa0c6 XL |
315 | return; |
316 | } | |
32a655c1 | 317 | |
a1dfa0c6 | 318 | "volatile_copy_nonoverlapping_memory" => { |
dfeec247 XL |
319 | copy_intrinsic( |
320 | self, | |
321 | false, | |
322 | true, | |
323 | substs.type_at(0), | |
324 | args[0].immediate(), | |
325 | args[1].immediate(), | |
326 | args[2].immediate(), | |
327 | ); | |
a1dfa0c6 XL |
328 | return; |
329 | } | |
330 | "volatile_copy_memory" => { | |
dfeec247 XL |
331 | copy_intrinsic( |
332 | self, | |
333 | true, | |
334 | true, | |
335 | substs.type_at(0), | |
336 | args[0].immediate(), | |
337 | args[1].immediate(), | |
338 | args[2].immediate(), | |
339 | ); | |
a1dfa0c6 XL |
340 | return; |
341 | } | |
342 | "volatile_set_memory" => { | |
dfeec247 XL |
343 | memset_intrinsic( |
344 | self, | |
345 | true, | |
346 | substs.type_at(0), | |
347 | args[0].immediate(), | |
348 | args[1].immediate(), | |
349 | args[2].immediate(), | |
350 | ); | |
a1dfa0c6 XL |
351 | return; |
352 | } | |
353 | "volatile_load" | "unaligned_volatile_load" => { | |
354 | let tp_ty = substs.type_at(0); | |
355 | let mut ptr = args[0].immediate(); | |
60c5eb7d | 356 | if let PassMode::Cast(ty) = fn_abi.ret.mode { |
a1dfa0c6 | 357 | ptr = self.pointercast(ptr, self.type_ptr_to(ty.llvm_type(self))); |
7453a54e | 358 | } |
a1dfa0c6 XL |
359 | let load = self.volatile_load(ptr); |
360 | let align = if name == "unaligned_volatile_load" { | |
361 | 1 | |
362 | } else { | |
363 | self.align_of(tp_ty).bytes() as u32 | |
364 | }; | |
365 | unsafe { | |
366 | llvm::LLVMSetAlignment(load, align); | |
367 | } | |
368 | to_immediate(self, load, self.layout_of(tp_ty)) | |
dfeec247 | 369 | } |
a1dfa0c6 XL |
370 | "volatile_store" => { |
371 | let dst = args[0].deref(self.cx()); | |
372 | args[1].val.volatile_store(self, dst); | |
373 | return; | |
dfeec247 | 374 | } |
a1dfa0c6 XL |
375 | "unaligned_volatile_store" => { |
376 | let dst = args[0].deref(self.cx()); | |
377 | args[1].val.unaligned_volatile_store(self, dst); | |
378 | return; | |
dfeec247 XL |
379 | } |
380 | "prefetch_read_data" | |
381 | | "prefetch_write_data" | |
382 | | "prefetch_read_instruction" | |
383 | | "prefetch_write_instruction" => { | |
a1dfa0c6 XL |
384 | let expect = self.get_intrinsic(&("llvm.prefetch")); |
385 | let (rw, cache_type) = match name { | |
386 | "prefetch_read_data" => (0, 1), | |
387 | "prefetch_write_data" => (1, 1), | |
388 | "prefetch_read_instruction" => (0, 0), | |
389 | "prefetch_write_instruction" => (1, 0), | |
dfeec247 | 390 | _ => bug!(), |
a1dfa0c6 | 391 | }; |
dfeec247 XL |
392 | self.call( |
393 | expect, | |
394 | &[ | |
395 | args[0].immediate(), | |
396 | self.const_i32(rw), | |
397 | args[1].immediate(), | |
398 | self.const_i32(cache_type), | |
399 | ], | |
400 | None, | |
401 | ) | |
402 | } | |
403 | "ctlz" | "ctlz_nonzero" | "cttz" | "cttz_nonzero" | "ctpop" | "bswap" | |
404 | | "bitreverse" | "add_with_overflow" | "sub_with_overflow" | "mul_with_overflow" | |
405 | | "wrapping_add" | "wrapping_sub" | "wrapping_mul" | "unchecked_div" | |
406 | | "unchecked_rem" | "unchecked_shl" | "unchecked_shr" | "unchecked_add" | |
407 | | "unchecked_sub" | "unchecked_mul" | "exact_div" | "rotate_left" | "rotate_right" | |
408 | | "saturating_add" | "saturating_sub" => { | |
a1dfa0c6 XL |
409 | let ty = arg_tys[0]; |
410 | match int_type_width_signed(ty, self) { | |
dfeec247 XL |
411 | Some((width, signed)) => match name { |
412 | "ctlz" | "cttz" => { | |
413 | let y = self.const_bool(false); | |
414 | let llfn = self.get_intrinsic(&format!("llvm.{}.i{}", name, width)); | |
415 | self.call(llfn, &[args[0].immediate(), y], None) | |
416 | } | |
417 | "ctlz_nonzero" | "cttz_nonzero" => { | |
418 | let y = self.const_bool(true); | |
419 | let llvm_name = &format!("llvm.{}.i{}", &name[..4], width); | |
420 | let llfn = self.get_intrinsic(llvm_name); | |
421 | self.call(llfn, &[args[0].immediate(), y], None) | |
422 | } | |
423 | "ctpop" => self.call( | |
424 | self.get_intrinsic(&format!("llvm.ctpop.i{}", width)), | |
425 | &[args[0].immediate()], | |
426 | None, | |
427 | ), | |
428 | "bswap" => { | |
429 | if width == 8 { | |
430 | args[0].immediate() // byte swap a u8/i8 is just a no-op | |
431 | } else { | |
a1dfa0c6 | 432 | self.call( |
dfeec247 | 433 | self.get_intrinsic(&format!("llvm.bswap.i{}", width)), |
a1dfa0c6 XL |
434 | &[args[0].immediate()], |
435 | None, | |
436 | ) | |
437 | } | |
dfeec247 XL |
438 | } |
439 | "bitreverse" => self.call( | |
440 | self.get_intrinsic(&format!("llvm.bitreverse.i{}", width)), | |
441 | &[args[0].immediate()], | |
442 | None, | |
443 | ), | |
444 | "add_with_overflow" | "sub_with_overflow" | "mul_with_overflow" => { | |
445 | let intrinsic = format!( | |
446 | "llvm.{}{}.with.overflow.i{}", | |
447 | if signed { 's' } else { 'u' }, | |
448 | &name[..3], | |
449 | width | |
450 | ); | |
451 | let llfn = self.get_intrinsic(&intrinsic); | |
452 | ||
453 | // Convert `i1` to a `bool`, and write it to the out parameter | |
454 | let pair = | |
455 | self.call(llfn, &[args[0].immediate(), args[1].immediate()], None); | |
456 | let val = self.extract_value(pair, 0); | |
457 | let overflow = self.extract_value(pair, 1); | |
458 | let overflow = self.zext(overflow, self.type_bool()); | |
459 | ||
460 | let dest = result.project_field(self, 0); | |
461 | self.store(val, dest.llval, dest.align); | |
462 | let dest = result.project_field(self, 1); | |
463 | self.store(overflow, dest.llval, dest.align); | |
464 | ||
465 | return; | |
466 | } | |
467 | "wrapping_add" => self.add(args[0].immediate(), args[1].immediate()), | |
468 | "wrapping_sub" => self.sub(args[0].immediate(), args[1].immediate()), | |
469 | "wrapping_mul" => self.mul(args[0].immediate(), args[1].immediate()), | |
470 | "exact_div" => { | |
471 | if signed { | |
472 | self.exactsdiv(args[0].immediate(), args[1].immediate()) | |
473 | } else { | |
474 | self.exactudiv(args[0].immediate(), args[1].immediate()) | |
475 | } | |
476 | } | |
477 | "unchecked_div" => { | |
478 | if signed { | |
479 | self.sdiv(args[0].immediate(), args[1].immediate()) | |
480 | } else { | |
481 | self.udiv(args[0].immediate(), args[1].immediate()) | |
482 | } | |
483 | } | |
484 | "unchecked_rem" => { | |
485 | if signed { | |
486 | self.srem(args[0].immediate(), args[1].immediate()) | |
487 | } else { | |
488 | self.urem(args[0].immediate(), args[1].immediate()) | |
489 | } | |
490 | } | |
491 | "unchecked_shl" => self.shl(args[0].immediate(), args[1].immediate()), | |
492 | "unchecked_shr" => { | |
493 | if signed { | |
494 | self.ashr(args[0].immediate(), args[1].immediate()) | |
495 | } else { | |
496 | self.lshr(args[0].immediate(), args[1].immediate()) | |
497 | } | |
498 | } | |
499 | "unchecked_add" => { | |
500 | if signed { | |
501 | self.unchecked_sadd(args[0].immediate(), args[1].immediate()) | |
502 | } else { | |
503 | self.unchecked_uadd(args[0].immediate(), args[1].immediate()) | |
504 | } | |
505 | } | |
506 | "unchecked_sub" => { | |
507 | if signed { | |
508 | self.unchecked_ssub(args[0].immediate(), args[1].immediate()) | |
509 | } else { | |
510 | self.unchecked_usub(args[0].immediate(), args[1].immediate()) | |
511 | } | |
512 | } | |
513 | "unchecked_mul" => { | |
514 | if signed { | |
515 | self.unchecked_smul(args[0].immediate(), args[1].immediate()) | |
516 | } else { | |
517 | self.unchecked_umul(args[0].immediate(), args[1].immediate()) | |
518 | } | |
519 | } | |
520 | "rotate_left" | "rotate_right" => { | |
521 | let is_left = name == "rotate_left"; | |
522 | let val = args[0].immediate(); | |
523 | let raw_shift = args[1].immediate(); | |
524 | // rotate = funnel shift with first two args the same | |
525 | let llvm_name = | |
526 | &format!("llvm.fsh{}.i{}", if is_left { 'l' } else { 'r' }, width); | |
527 | let llfn = self.get_intrinsic(llvm_name); | |
528 | self.call(llfn, &[val, val, raw_shift], None) | |
529 | } | |
530 | "saturating_add" | "saturating_sub" => { | |
531 | let is_add = name == "saturating_add"; | |
532 | let lhs = args[0].immediate(); | |
533 | let rhs = args[1].immediate(); | |
ba9703b0 XL |
534 | let llvm_name = &format!( |
535 | "llvm.{}{}.sat.i{}", | |
536 | if signed { 's' } else { 'u' }, | |
537 | if is_add { "add" } else { "sub" }, | |
538 | width | |
539 | ); | |
540 | let llfn = self.get_intrinsic(llvm_name); | |
541 | self.call(llfn, &[lhs, rhs], None) | |
dfeec247 XL |
542 | } |
543 | _ => bug!(), | |
544 | }, | |
a1dfa0c6 XL |
545 | None => { |
546 | span_invalid_monomorphization_error( | |
dfeec247 XL |
547 | tcx.sess, |
548 | span, | |
549 | &format!( | |
550 | "invalid monomorphization of `{}` intrinsic: \ | |
551 | expected basic integer type, found `{}`", | |
552 | name, ty | |
553 | ), | |
554 | ); | |
a1dfa0c6 | 555 | return; |
54a0048b | 556 | } |
1a4d82fc | 557 | } |
dfeec247 | 558 | } |
a1dfa0c6 | 559 | "fadd_fast" | "fsub_fast" | "fmul_fast" | "fdiv_fast" | "frem_fast" => { |
532ac7d7 | 560 | match float_type_width(arg_tys[0]) { |
dfeec247 XL |
561 | Some(_width) => match name { |
562 | "fadd_fast" => self.fadd_fast(args[0].immediate(), args[1].immediate()), | |
563 | "fsub_fast" => self.fsub_fast(args[0].immediate(), args[1].immediate()), | |
564 | "fmul_fast" => self.fmul_fast(args[0].immediate(), args[1].immediate()), | |
565 | "fdiv_fast" => self.fdiv_fast(args[0].immediate(), args[1].immediate()), | |
566 | "frem_fast" => self.frem_fast(args[0].immediate(), args[1].immediate()), | |
567 | _ => bug!(), | |
568 | }, | |
a1dfa0c6 XL |
569 | None => { |
570 | span_invalid_monomorphization_error( | |
dfeec247 XL |
571 | tcx.sess, |
572 | span, | |
573 | &format!( | |
574 | "invalid monomorphization of `{}` intrinsic: \ | |
575 | expected basic float type, found `{}`", | |
576 | name, arg_tys[0] | |
577 | ), | |
578 | ); | |
ff7c6d11 | 579 | return; |
54a0048b | 580 | } |
1a4d82fc | 581 | } |
dfeec247 | 582 | } |
d9579d0f | 583 | |
ba9703b0 | 584 | "float_to_int_unchecked" => { |
60c5eb7d XL |
585 | if float_type_width(arg_tys[0]).is_none() { |
586 | span_invalid_monomorphization_error( | |
dfeec247 XL |
587 | tcx.sess, |
588 | span, | |
589 | &format!( | |
ba9703b0 | 590 | "invalid monomorphization of `float_to_int_unchecked` \ |
60c5eb7d | 591 | intrinsic: expected basic float type, \ |
dfeec247 XL |
592 | found `{}`", |
593 | arg_tys[0] | |
594 | ), | |
595 | ); | |
60c5eb7d XL |
596 | return; |
597 | } | |
598 | match int_type_width_signed(ret_ty, self.cx) { | |
599 | Some((width, signed)) => { | |
600 | if signed { | |
601 | self.fptosi(args[0].immediate(), self.cx.type_ix(width)) | |
602 | } else { | |
603 | self.fptoui(args[0].immediate(), self.cx.type_ix(width)) | |
604 | } | |
605 | } | |
606 | None => { | |
607 | span_invalid_monomorphization_error( | |
dfeec247 XL |
608 | tcx.sess, |
609 | span, | |
610 | &format!( | |
ba9703b0 | 611 | "invalid monomorphization of `float_to_int_unchecked` \ |
60c5eb7d | 612 | intrinsic: expected basic integer type, \ |
dfeec247 XL |
613 | found `{}`", |
614 | ret_ty | |
615 | ), | |
616 | ); | |
60c5eb7d XL |
617 | return; |
618 | } | |
619 | } | |
620 | } | |
621 | ||
f9f354fc XL |
622 | "discriminant_value" => { |
623 | if ret_ty.is_integral() { | |
624 | args[0].deref(self.cx()).codegen_get_discr(self, ret_ty) | |
625 | } else { | |
626 | span_bug!(span, "Invalid discriminant type for `{:?}`", arg_tys[0]) | |
627 | } | |
628 | } | |
1a4d82fc | 629 | |
a1dfa0c6 | 630 | name if name.starts_with("simd_") => { |
dfeec247 | 631 | match generic_simd_intrinsic(self, name, callee_ty, args, ret_ty, llret_ty, span) { |
a1dfa0c6 | 632 | Ok(llval) => llval, |
dfeec247 | 633 | Err(()) => return, |
1a4d82fc JJ |
634 | } |
635 | } | |
a1dfa0c6 XL |
636 | // This requires that atomic intrinsics follow a specific naming pattern: |
637 | // "atomic_<operation>[_<ordering>]", and no ordering means SeqCst | |
638 | name if name.starts_with("atomic_") => { | |
639 | use rustc_codegen_ssa::common::AtomicOrdering::*; | |
dfeec247 | 640 | use rustc_codegen_ssa::common::{AtomicRmwBinOp, SynchronizationScope}; |
a1dfa0c6 XL |
641 | |
642 | let split: Vec<&str> = name.split('_').collect(); | |
643 | ||
644 | let is_cxchg = split[1] == "cxchg" || split[1] == "cxchgweak"; | |
645 | let (order, failorder) = match split.len() { | |
646 | 2 => (SequentiallyConsistent, SequentiallyConsistent), | |
647 | 3 => match split[2] { | |
648 | "unordered" => (Unordered, Unordered), | |
649 | "relaxed" => (Monotonic, Monotonic), | |
dfeec247 XL |
650 | "acq" => (Acquire, Acquire), |
651 | "rel" => (Release, Monotonic), | |
652 | "acqrel" => (AcquireRelease, Acquire), | |
653 | "failrelaxed" if is_cxchg => (SequentiallyConsistent, Monotonic), | |
654 | "failacq" if is_cxchg => (SequentiallyConsistent, Acquire), | |
655 | _ => self.sess().fatal("unknown ordering in atomic intrinsic"), | |
a1dfa0c6 XL |
656 | }, |
657 | 4 => match (split[2], split[3]) { | |
dfeec247 XL |
658 | ("acq", "failrelaxed") if is_cxchg => (Acquire, Monotonic), |
659 | ("acqrel", "failrelaxed") if is_cxchg => (AcquireRelease, Monotonic), | |
660 | _ => self.sess().fatal("unknown ordering in atomic intrinsic"), | |
a1dfa0c6 XL |
661 | }, |
662 | _ => self.sess().fatal("Atomic intrinsic not in correct format"), | |
663 | }; | |
ff7c6d11 | 664 | |
a1dfa0c6 | 665 | let invalid_monomorphization = |ty| { |
dfeec247 XL |
666 | span_invalid_monomorphization_error( |
667 | tcx.sess, | |
668 | span, | |
669 | &format!( | |
670 | "invalid monomorphization of `{}` intrinsic: \ | |
671 | expected basic integer type, found `{}`", | |
672 | name, ty | |
673 | ), | |
674 | ); | |
a1dfa0c6 | 675 | }; |
0bf4aa26 | 676 | |
a1dfa0c6 XL |
677 | match split[1] { |
678 | "cxchg" | "cxchgweak" => { | |
679 | let ty = substs.type_at(0); | |
680 | if int_type_width_signed(ty, self).is_some() { | |
681 | let weak = split[1] == "cxchgweak"; | |
682 | let pair = self.atomic_cmpxchg( | |
683 | args[0].immediate(), | |
684 | args[1].immediate(), | |
685 | args[2].immediate(), | |
686 | order, | |
687 | failorder, | |
dfeec247 XL |
688 | weak, |
689 | ); | |
a1dfa0c6 XL |
690 | let val = self.extract_value(pair, 0); |
691 | let success = self.extract_value(pair, 1); | |
692 | let success = self.zext(success, self.type_bool()); | |
693 | ||
694 | let dest = result.project_field(self, 0); | |
695 | self.store(val, dest.llval, dest.align); | |
696 | let dest = result.project_field(self, 1); | |
697 | self.store(success, dest.llval, dest.align); | |
698 | return; | |
699 | } else { | |
700 | return invalid_monomorphization(ty); | |
701 | } | |
e9174d1e | 702 | } |
a1dfa0c6 XL |
703 | |
704 | "load" => { | |
705 | let ty = substs.type_at(0); | |
706 | if int_type_width_signed(ty, self).is_some() { | |
707 | let size = self.size_of(ty); | |
708 | self.atomic_load(args[0].immediate(), order, size) | |
709 | } else { | |
710 | return invalid_monomorphization(ty); | |
e9174d1e SL |
711 | } |
712 | } | |
a1dfa0c6 XL |
713 | |
714 | "store" => { | |
715 | let ty = substs.type_at(0); | |
716 | if int_type_width_signed(ty, self).is_some() { | |
717 | let size = self.size_of(ty); | |
718 | self.atomic_store( | |
719 | args[1].immediate(), | |
720 | args[0].immediate(), | |
721 | order, | |
dfeec247 | 722 | size, |
a1dfa0c6 XL |
723 | ); |
724 | return; | |
725 | } else { | |
726 | return invalid_monomorphization(ty); | |
727 | } | |
e9174d1e | 728 | } |
a1dfa0c6 XL |
729 | |
730 | "fence" => { | |
731 | self.atomic_fence(order, SynchronizationScope::CrossThread); | |
732 | return; | |
e9174d1e | 733 | } |
a1dfa0c6 XL |
734 | |
735 | "singlethreadfence" => { | |
736 | self.atomic_fence(order, SynchronizationScope::SingleThread); | |
737 | return; | |
e9174d1e | 738 | } |
a1dfa0c6 XL |
739 | |
740 | // These are all AtomicRMW ops | |
741 | op => { | |
742 | let atom_op = match op { | |
dfeec247 XL |
743 | "xchg" => AtomicRmwBinOp::AtomicXchg, |
744 | "xadd" => AtomicRmwBinOp::AtomicAdd, | |
745 | "xsub" => AtomicRmwBinOp::AtomicSub, | |
746 | "and" => AtomicRmwBinOp::AtomicAnd, | |
747 | "nand" => AtomicRmwBinOp::AtomicNand, | |
748 | "or" => AtomicRmwBinOp::AtomicOr, | |
749 | "xor" => AtomicRmwBinOp::AtomicXor, | |
750 | "max" => AtomicRmwBinOp::AtomicMax, | |
751 | "min" => AtomicRmwBinOp::AtomicMin, | |
752 | "umax" => AtomicRmwBinOp::AtomicUMax, | |
753 | "umin" => AtomicRmwBinOp::AtomicUMin, | |
754 | _ => self.sess().fatal("unknown atomic operation"), | |
a1dfa0c6 XL |
755 | }; |
756 | ||
757 | let ty = substs.type_at(0); | |
758 | if int_type_width_signed(ty, self).is_some() { | |
759 | self.atomic_rmw( | |
760 | atom_op, | |
761 | args[0].immediate(), | |
762 | args[1].immediate(), | |
dfeec247 | 763 | order, |
a1dfa0c6 XL |
764 | ) |
765 | } else { | |
766 | return invalid_monomorphization(ty); | |
767 | } | |
e9174d1e SL |
768 | } |
769 | } | |
770 | } | |
771 | ||
a1dfa0c6 XL |
772 | "nontemporal_store" => { |
773 | let dst = args[0].deref(self.cx()); | |
774 | args[1].val.nontemporal_store(self, dst); | |
775 | return; | |
776 | } | |
777 | ||
f035d41b XL |
778 | "ptr_guaranteed_eq" | "ptr_guaranteed_ne" => { |
779 | let a = args[0].immediate(); | |
780 | let b = args[1].immediate(); | |
781 | if name == "ptr_guaranteed_eq" { | |
782 | self.icmp(IntPredicate::IntEQ, a, b) | |
783 | } else { | |
784 | self.icmp(IntPredicate::IntNE, a, b) | |
785 | } | |
786 | } | |
787 | ||
e74abb32 XL |
788 | "ptr_offset_from" => { |
789 | let ty = substs.type_at(0); | |
790 | let pointee_size = self.size_of(ty); | |
791 | ||
792 | // This is the same sequence that Clang emits for pointer subtraction. | |
793 | // It can be neither `nsw` nor `nuw` because the input is treated as | |
794 | // unsigned but then the output is treated as signed, so neither works. | |
795 | let a = args[0].immediate(); | |
796 | let b = args[1].immediate(); | |
797 | let a = self.ptrtoint(a, self.type_isize()); | |
798 | let b = self.ptrtoint(b, self.type_isize()); | |
799 | let d = self.sub(a, b); | |
800 | let pointee_size = self.const_usize(pointee_size.bytes()); | |
801 | // this is where the signed magic happens (notice the `s` in `exactsdiv`) | |
802 | self.exactsdiv(d, pointee_size) | |
803 | } | |
804 | ||
0731742a | 805 | _ => bug!("unknown intrinsic '{}'", name), |
a1dfa0c6 | 806 | }; |
1a4d82fc | 807 | |
60c5eb7d XL |
808 | if !fn_abi.ret.is_ignore() { |
809 | if let PassMode::Cast(ty) = fn_abi.ret.mode { | |
a1dfa0c6 XL |
810 | let ptr_llty = self.type_ptr_to(ty.llvm_type(self)); |
811 | let ptr = self.pointercast(result.llval, ptr_llty); | |
812 | self.store(llval, ptr, result.align); | |
813 | } else { | |
814 | OperandRef::from_immediate_or_packed_pair(self, llval, result.layout) | |
dfeec247 XL |
815 | .val |
816 | .store(self, result); | |
a1dfa0c6 | 817 | } |
54a0048b | 818 | } |
1a4d82fc | 819 | } |
a1dfa0c6 XL |
820 | |
821 | fn abort(&mut self) { | |
822 | let fnname = self.get_intrinsic(&("llvm.trap")); | |
823 | self.call(fnname, &[], None); | |
824 | } | |
825 | ||
826 | fn assume(&mut self, val: Self::Value) { | |
827 | let assume_intrinsic = self.get_intrinsic("llvm.assume"); | |
828 | self.call(assume_intrinsic, &[val], None); | |
829 | } | |
830 | ||
831 | fn expect(&mut self, cond: Self::Value, expected: bool) -> Self::Value { | |
832 | let expect = self.get_intrinsic(&"llvm.expect.i1"); | |
833 | self.call(expect, &[cond, self.const_bool(expected)], None) | |
834 | } | |
532ac7d7 | 835 | |
e74abb32 XL |
836 | fn sideeffect(&mut self) { |
837 | if self.tcx.sess.opts.debugging_opts.insert_sideeffect { | |
838 | let fnname = self.get_intrinsic(&("llvm.sideeffect")); | |
839 | self.call(fnname, &[], None); | |
840 | } | |
841 | } | |
842 | ||
dc9dc135 | 843 | fn va_start(&mut self, va_list: &'ll Value) -> &'ll Value { |
532ac7d7 XL |
844 | let intrinsic = self.cx().get_intrinsic("llvm.va_start"); |
845 | self.call(intrinsic, &[va_list], None) | |
846 | } | |
847 | ||
dc9dc135 | 848 | fn va_end(&mut self, va_list: &'ll Value) -> &'ll Value { |
532ac7d7 XL |
849 | let intrinsic = self.cx().get_intrinsic("llvm.va_end"); |
850 | self.call(intrinsic, &[va_list], None) | |
851 | } | |
1a4d82fc JJ |
852 | } |
853 | ||
b7449926 | 854 | fn copy_intrinsic( |
a1dfa0c6 | 855 | bx: &mut Builder<'a, 'll, 'tcx>, |
b7449926 XL |
856 | allow_overlap: bool, |
857 | volatile: bool, | |
858 | ty: Ty<'tcx>, | |
859 | dst: &'ll Value, | |
860 | src: &'ll Value, | |
861 | count: &'ll Value, | |
a1dfa0c6 XL |
862 | ) { |
863 | let (size, align) = bx.size_and_align_of(ty); | |
864 | let size = bx.mul(bx.const_usize(size.bytes()), count); | |
dfeec247 | 865 | let flags = if volatile { MemFlags::VOLATILE } else { MemFlags::empty() }; |
a1dfa0c6 XL |
866 | if allow_overlap { |
867 | bx.memmove(dst, align, src, align, size, flags); | |
868 | } else { | |
869 | bx.memcpy(dst, align, src, align, size, flags); | |
870 | } | |
1a4d82fc JJ |
871 | } |
872 | ||
b7449926 | 873 | fn memset_intrinsic( |
a1dfa0c6 | 874 | bx: &mut Builder<'a, 'll, 'tcx>, |
32a655c1 SL |
875 | volatile: bool, |
876 | ty: Ty<'tcx>, | |
b7449926 XL |
877 | dst: &'ll Value, |
878 | val: &'ll Value, | |
dfeec247 | 879 | count: &'ll Value, |
a1dfa0c6 XL |
880 | ) { |
881 | let (size, align) = bx.size_and_align_of(ty); | |
882 | let size = bx.mul(bx.const_usize(size.bytes()), count); | |
dfeec247 | 883 | let flags = if volatile { MemFlags::VOLATILE } else { MemFlags::empty() }; |
a1dfa0c6 | 884 | bx.memset(dst, val, size, align, flags); |
1a4d82fc | 885 | } |
c1a9b12d | 886 | |
b7449926 | 887 | fn try_intrinsic( |
a1dfa0c6 | 888 | bx: &mut Builder<'a, 'll, 'tcx>, |
ba9703b0 | 889 | try_func: &'ll Value, |
b7449926 | 890 | data: &'ll Value, |
ba9703b0 | 891 | catch_func: &'ll Value, |
b7449926 | 892 | dest: &'ll Value, |
32a655c1 | 893 | ) { |
f9f354fc | 894 | if bx.sess().panic_strategy() == PanicStrategy::Abort { |
ba9703b0 XL |
895 | bx.call(try_func, &[data], None); |
896 | // Return 0 unconditionally from the intrinsic call; | |
897 | // we can never unwind. | |
898 | let ret_align = bx.tcx().data_layout.i32_align.abi; | |
899 | bx.store(bx.const_i32(0), dest, ret_align); | |
2c00a5a8 | 900 | } else if wants_msvc_seh(bx.sess()) { |
ba9703b0 | 901 | codegen_msvc_try(bx, try_func, data, catch_func, dest); |
c1a9b12d | 902 | } else { |
ba9703b0 | 903 | codegen_gnu_try(bx, try_func, data, catch_func, dest); |
c1a9b12d SL |
904 | } |
905 | } | |
906 | ||
7453a54e | 907 | // MSVC's definition of the `rust_try` function. |
c1a9b12d | 908 | // |
7453a54e SL |
909 | // This implementation uses the new exception handling instructions in LLVM |
910 | // which have support in LLVM for SEH on MSVC targets. Although these | |
911 | // instructions are meant to work for all targets, as of the time of this | |
912 | // writing, however, LLVM does not recommend the usage of these new instructions | |
913 | // as the old ones are still more optimized. | |
b7449926 | 914 | fn codegen_msvc_try( |
a1dfa0c6 | 915 | bx: &mut Builder<'a, 'll, 'tcx>, |
ba9703b0 | 916 | try_func: &'ll Value, |
b7449926 | 917 | data: &'ll Value, |
ba9703b0 | 918 | catch_func: &'ll Value, |
b7449926 XL |
919 | dest: &'ll Value, |
920 | ) { | |
a1dfa0c6 XL |
921 | let llfn = get_rust_try_fn(bx, &mut |mut bx| { |
922 | bx.set_personality_fn(bx.eh_personality()); | |
e74abb32 | 923 | bx.sideeffect(); |
32a655c1 | 924 | |
a1dfa0c6 XL |
925 | let mut normal = bx.build_sibling_block("normal"); |
926 | let mut catchswitch = bx.build_sibling_block("catchswitch"); | |
927 | let mut catchpad = bx.build_sibling_block("catchpad"); | |
928 | let mut caught = bx.build_sibling_block("caught"); | |
32a655c1 | 929 | |
ba9703b0 | 930 | let try_func = llvm::get_param(bx.llfn(), 0); |
2c00a5a8 | 931 | let data = llvm::get_param(bx.llfn(), 1); |
ba9703b0 | 932 | let catch_func = llvm::get_param(bx.llfn(), 2); |
c1a9b12d | 933 | |
7453a54e SL |
934 | // We're generating an IR snippet that looks like: |
935 | // | |
ba9703b0 XL |
936 | // declare i32 @rust_try(%try_func, %data, %catch_func) { |
937 | // %slot = alloca u8* | |
938 | // invoke %try_func(%data) to label %normal unwind label %catchswitch | |
7453a54e SL |
939 | // |
940 | // normal: | |
941 | // ret i32 0 | |
c1a9b12d | 942 | // |
7453a54e SL |
943 | // catchswitch: |
944 | // %cs = catchswitch within none [%catchpad] unwind to caller | |
c1a9b12d | 945 | // |
7453a54e | 946 | // catchpad: |
a7813a04 | 947 | // %tok = catchpad within %cs [%type_descriptor, 0, %slot] |
ba9703b0 XL |
948 | // %ptr = load %slot |
949 | // call %catch_func(%data, %ptr) | |
7453a54e | 950 | // catchret from %tok to label %caught |
c1a9b12d | 951 | // |
7453a54e SL |
952 | // caught: |
953 | // ret i32 1 | |
954 | // } | |
c1a9b12d | 955 | // |
a7813a04 XL |
956 | // This structure follows the basic usage of throw/try/catch in LLVM. |
957 | // For example, compile this C++ snippet to see what LLVM generates: | |
958 | // | |
959 | // #include <stdint.h> | |
960 | // | |
e74abb32 | 961 | // struct rust_panic { |
dfeec247 XL |
962 | // rust_panic(const rust_panic&); |
963 | // ~rust_panic(); | |
964 | // | |
e74abb32 | 965 | // uint64_t x[2]; |
ba9703b0 | 966 | // }; |
e74abb32 | 967 | // |
ba9703b0 XL |
968 | // int __rust_try( |
969 | // void (*try_func)(void*), | |
970 | // void *data, | |
971 | // void (*catch_func)(void*, void*) noexcept | |
972 | // ) { | |
a7813a04 | 973 | // try { |
ba9703b0 | 974 | // try_func(data); |
a7813a04 | 975 | // return 0; |
dfeec247 | 976 | // } catch(rust_panic& a) { |
ba9703b0 | 977 | // catch_func(data, &a); |
a7813a04 XL |
978 | // return 1; |
979 | // } | |
980 | // } | |
7453a54e SL |
981 | // |
982 | // More information can be found in libstd's seh.rs implementation. | |
dfeec247 | 983 | let ptr_align = bx.tcx().data_layout.pointer_align.abi; |
ba9703b0 XL |
984 | let slot = bx.alloca(bx.type_i8p(), ptr_align); |
985 | bx.invoke(try_func, &[data], normal.llbb(), catchswitch.llbb(), None); | |
7453a54e | 986 | |
a1dfa0c6 | 987 | normal.ret(bx.const_i32(0)); |
7453a54e | 988 | |
32a655c1 SL |
989 | let cs = catchswitch.catch_switch(None, None, 1); |
990 | catchswitch.add_handler(cs, catchpad.llbb()); | |
7453a54e | 991 | |
ba9703b0 XL |
992 | // We can't use the TypeDescriptor defined in libpanic_unwind because it |
993 | // might be in another DLL and the SEH encoding only supports specifying | |
994 | // a TypeDescriptor from the current module. | |
995 | // | |
996 | // However this isn't an issue since the MSVC runtime uses string | |
997 | // comparison on the type name to match TypeDescriptors rather than | |
998 | // pointer equality. | |
999 | // | |
1000 | // So instead we generate a new TypeDescriptor in each module that uses | |
1001 | // `try` and let the linker merge duplicate definitions in the same | |
1002 | // module. | |
1003 | // | |
1004 | // When modifying, make sure that the type_name string exactly matches | |
1005 | // the one used in src/libpanic_unwind/seh.rs. | |
1006 | let type_info_vtable = bx.declare_global("??_7type_info@@6B@", bx.type_i8p()); | |
1007 | let type_name = bx.const_bytes(b"rust_panic\0"); | |
1008 | let type_info = | |
1009 | bx.const_struct(&[type_info_vtable, bx.const_null(bx.type_i8p()), type_name], false); | |
1010 | let tydesc = bx.declare_global("__rust_panic_type_info", bx.val_ty(type_info)); | |
1011 | unsafe { | |
1012 | llvm::LLVMRustSetLinkage(tydesc, llvm::Linkage::LinkOnceODRLinkage); | |
1013 | llvm::SetUniqueComdat(bx.llmod, tydesc); | |
1014 | llvm::LLVMSetInitializer(tydesc, type_info); | |
1015 | } | |
1016 | ||
dfeec247 XL |
1017 | // The flag value of 8 indicates that we are catching the exception by |
1018 | // reference instead of by value. We can't use catch by value because | |
1019 | // that requires copying the exception object, which we don't support | |
1020 | // since our exception object effectively contains a Box. | |
1021 | // | |
1022 | // Source: MicrosoftCXXABI::getAddrOfCXXCatchHandlerType in clang | |
1023 | let flags = bx.const_i32(8); | |
dfeec247 | 1024 | let funclet = catchpad.catch_pad(cs, &[tydesc, flags, slot]); |
ba9703b0 XL |
1025 | let ptr = catchpad.load(slot, ptr_align); |
1026 | catchpad.call(catch_func, &[data, ptr], Some(&funclet)); | |
dfeec247 | 1027 | |
a1dfa0c6 | 1028 | catchpad.catch_ret(&funclet, caught.llbb()); |
7453a54e | 1029 | |
a1dfa0c6 | 1030 | caught.ret(bx.const_i32(1)); |
c1a9b12d SL |
1031 | }); |
1032 | ||
1033 | // Note that no invoke is used here because by definition this function | |
1034 | // can't panic (that's what it's catching). | |
ba9703b0 | 1035 | let ret = bx.call(llfn, &[try_func, data, catch_func], None); |
a1dfa0c6 | 1036 | let i32_align = bx.tcx().data_layout.i32_align.abi; |
2c00a5a8 | 1037 | bx.store(ret, dest, i32_align); |
c1a9b12d SL |
1038 | } |
1039 | ||
dc9dc135 XL |
1040 | // Definition of the standard `try` function for Rust using the GNU-like model |
1041 | // of exceptions (e.g., the normal semantics of LLVM's `landingpad` and `invoke` | |
c1a9b12d SL |
1042 | // instructions). |
1043 | // | |
94b46f34 | 1044 | // This codegen is a little surprising because we always call a shim |
7453a54e SL |
1045 | // function instead of inlining the call to `invoke` manually here. This is done |
1046 | // because in LLVM we're only allowed to have one personality per function | |
1047 | // definition. The call to the `try` intrinsic is being inlined into the | |
1048 | // function calling it, and that function may already have other personality | |
1049 | // functions in play. By calling a shim we're guaranteed that our shim will have | |
1050 | // the right personality function. | |
b7449926 | 1051 | fn codegen_gnu_try( |
a1dfa0c6 | 1052 | bx: &mut Builder<'a, 'll, 'tcx>, |
ba9703b0 | 1053 | try_func: &'ll Value, |
b7449926 | 1054 | data: &'ll Value, |
ba9703b0 | 1055 | catch_func: &'ll Value, |
b7449926 XL |
1056 | dest: &'ll Value, |
1057 | ) { | |
a1dfa0c6 | 1058 | let llfn = get_rust_try_fn(bx, &mut |mut bx| { |
94b46f34 | 1059 | // Codegens the shims described above: |
c1a9b12d | 1060 | // |
2c00a5a8 | 1061 | // bx: |
ba9703b0 | 1062 | // invoke %try_func(%data) normal %normal unwind %catch |
c1a9b12d SL |
1063 | // |
1064 | // normal: | |
7453a54e | 1065 | // ret 0 |
c1a9b12d SL |
1066 | // |
1067 | // catch: | |
ba9703b0 XL |
1068 | // (%ptr, _) = landingpad |
1069 | // call %catch_func(%data, %ptr) | |
7453a54e | 1070 | // ret 1 |
c1a9b12d | 1071 | |
e74abb32 XL |
1072 | bx.sideeffect(); |
1073 | ||
a1dfa0c6 XL |
1074 | let mut then = bx.build_sibling_block("then"); |
1075 | let mut catch = bx.build_sibling_block("catch"); | |
c1a9b12d | 1076 | |
ba9703b0 | 1077 | let try_func = llvm::get_param(bx.llfn(), 0); |
2c00a5a8 | 1078 | let data = llvm::get_param(bx.llfn(), 1); |
ba9703b0 XL |
1079 | let catch_func = llvm::get_param(bx.llfn(), 2); |
1080 | bx.invoke(try_func, &[data], then.llbb(), catch.llbb(), None); | |
a1dfa0c6 | 1081 | then.ret(bx.const_i32(0)); |
c1a9b12d SL |
1082 | |
1083 | // Type indicator for the exception being thrown. | |
7453a54e SL |
1084 | // |
1085 | // The first value in this tuple is a pointer to the exception object | |
1086 | // being thrown. The second value is a "selector" indicating which of | |
1087 | // the landing pad clauses the exception's type had been matched to. | |
1088 | // rust_try ignores the selector. | |
a1dfa0c6 XL |
1089 | let lpad_ty = bx.type_struct(&[bx.type_i8p(), bx.type_i32()], false); |
1090 | let vals = catch.landing_pad(lpad_ty, bx.eh_personality(), 1); | |
e74abb32 XL |
1091 | let tydesc = match bx.tcx().lang_items().eh_catch_typeinfo() { |
1092 | Some(tydesc) => { | |
1093 | let tydesc = bx.get_static(tydesc); | |
1094 | bx.bitcast(tydesc, bx.type_i8p()) | |
1095 | } | |
1096 | None => bx.const_null(bx.type_i8p()), | |
1097 | }; | |
1098 | catch.add_clause(vals, tydesc); | |
32a655c1 | 1099 | let ptr = catch.extract_value(vals, 0); |
ba9703b0 | 1100 | catch.call(catch_func, &[data, ptr], None); |
a1dfa0c6 | 1101 | catch.ret(bx.const_i32(1)); |
c1a9b12d SL |
1102 | }); |
1103 | ||
1104 | // Note that no invoke is used here because by definition this function | |
1105 | // can't panic (that's what it's catching). | |
ba9703b0 | 1106 | let ret = bx.call(llfn, &[try_func, data, catch_func], None); |
a1dfa0c6 | 1107 | let i32_align = bx.tcx().data_layout.i32_align.abi; |
2c00a5a8 | 1108 | bx.store(ret, dest, i32_align); |
c1a9b12d SL |
1109 | } |
1110 | ||
94b46f34 | 1111 | // Helper function to give a Block to a closure to codegen a shim function. |
7453a54e | 1112 | // This is currently primarily used for the `try` intrinsic functions above. |
b7449926 XL |
1113 | fn gen_fn<'ll, 'tcx>( |
1114 | cx: &CodegenCx<'ll, 'tcx>, | |
1115 | name: &str, | |
1116 | inputs: Vec<Ty<'tcx>>, | |
1117 | output: Ty<'tcx>, | |
1118 | codegen: &mut dyn FnMut(Builder<'_, 'll, 'tcx>), | |
1119 | ) -> &'ll Value { | |
a1dfa0c6 | 1120 | let rust_fn_sig = ty::Binder::bind(cx.tcx.mk_fn_sig( |
8bb4bdeb XL |
1121 | inputs.into_iter(), |
1122 | output, | |
1123 | false, | |
1124 | hir::Unsafety::Unsafe, | |
dfeec247 | 1125 | Abi::Rust, |
a1dfa0c6 | 1126 | )); |
60c5eb7d XL |
1127 | let fn_abi = FnAbi::of_fn_ptr(cx, rust_fn_sig, &[]); |
1128 | let llfn = cx.declare_fn(name, &fn_abi); | |
ba9703b0 XL |
1129 | cx.set_frame_pointer_elimination(llfn); |
1130 | cx.apply_target_cpu_attr(llfn); | |
60c5eb7d XL |
1131 | // FIXME(eddyb) find a nicer way to do this. |
1132 | unsafe { llvm::LLVMRustSetLinkage(llfn, llvm::Linkage::InternalLinkage) }; | |
2c00a5a8 | 1133 | let bx = Builder::new_block(cx, llfn, "entry-block"); |
94b46f34 | 1134 | codegen(bx); |
54a0048b | 1135 | llfn |
7453a54e SL |
1136 | } |
1137 | ||
1138 | // Helper function used to get a handle to the `__rust_try` function used to | |
1139 | // catch exceptions. | |
1140 | // | |
1141 | // This function is only generated once and is then cached. | |
b7449926 XL |
1142 | fn get_rust_try_fn<'ll, 'tcx>( |
1143 | cx: &CodegenCx<'ll, 'tcx>, | |
1144 | codegen: &mut dyn FnMut(Builder<'_, 'll, 'tcx>), | |
1145 | ) -> &'ll Value { | |
2c00a5a8 | 1146 | if let Some(llfn) = cx.rust_try_fn.get() { |
54a0048b | 1147 | return llfn; |
c1a9b12d SL |
1148 | } |
1149 | ||
1150 | // Define the type up front for the signature of the rust_try function. | |
2c00a5a8 | 1151 | let tcx = cx.tcx; |
c1a9b12d | 1152 | let i8p = tcx.mk_mut_ptr(tcx.types.i8); |
ba9703b0 | 1153 | let try_fn_ty = tcx.mk_fn_ptr(ty::Binder::bind(tcx.mk_fn_sig( |
8bb4bdeb | 1154 | iter::once(i8p), |
b7449926 | 1155 | tcx.mk_unit(), |
8bb4bdeb XL |
1156 | false, |
1157 | hir::Unsafety::Unsafe, | |
dfeec247 | 1158 | Abi::Rust, |
8bb4bdeb | 1159 | ))); |
ba9703b0 XL |
1160 | let catch_fn_ty = tcx.mk_fn_ptr(ty::Binder::bind(tcx.mk_fn_sig( |
1161 | [i8p, i8p].iter().cloned(), | |
1162 | tcx.mk_unit(), | |
1163 | false, | |
1164 | hir::Unsafety::Unsafe, | |
1165 | Abi::Rust, | |
1166 | ))); | |
5bcae85e | 1167 | let output = tcx.types.i32; |
ba9703b0 | 1168 | let rust_try = gen_fn(cx, "__rust_try", vec![try_fn_ty, i8p, catch_fn_ty], output, codegen); |
2c00a5a8 | 1169 | cx.rust_try_fn.set(Some(rust_try)); |
0bf4aa26 | 1170 | rust_try |
c1a9b12d | 1171 | } |
e9174d1e | 1172 | |
b7449926 | 1173 | fn generic_simd_intrinsic( |
a1dfa0c6 | 1174 | bx: &mut Builder<'a, 'll, 'tcx>, |
32a655c1 SL |
1175 | name: &str, |
1176 | callee_ty: Ty<'tcx>, | |
a1dfa0c6 | 1177 | args: &[OperandRef<'tcx, &'ll Value>], |
32a655c1 | 1178 | ret_ty: Ty<'tcx>, |
b7449926 | 1179 | llret_ty: &'ll Type, |
dfeec247 | 1180 | span: Span, |
b7449926 | 1181 | ) -> Result<&'ll Value, ()> { |
e9174d1e SL |
1182 | // macros for error handling: |
1183 | macro_rules! emit_error { | |
1184 | ($msg: tt) => { | |
1185 | emit_error!($msg, ) | |
1186 | }; | |
1187 | ($msg: tt, $($fmt: tt)*) => { | |
b039eaaf | 1188 | span_invalid_monomorphization_error( |
2c00a5a8 | 1189 | bx.sess(), span, |
0bf4aa26 | 1190 | &format!(concat!("invalid monomorphization of `{}` intrinsic: ", $msg), |
b039eaaf | 1191 | name, $($fmt)*)); |
e9174d1e SL |
1192 | } |
1193 | } | |
0bf4aa26 | 1194 | |
0531ce1d XL |
1195 | macro_rules! return_error { |
1196 | ($($fmt: tt)*) => { | |
1197 | { | |
e9174d1e | 1198 | emit_error!($($fmt)*); |
ff7c6d11 | 1199 | return Err(()); |
e9174d1e SL |
1200 | } |
1201 | } | |
1202 | } | |
0531ce1d XL |
1203 | |
1204 | macro_rules! require { | |
1205 | ($cond: expr, $($fmt: tt)*) => { | |
1206 | if !$cond { | |
1207 | return_error!($($fmt)*); | |
1208 | } | |
1209 | }; | |
1210 | } | |
0bf4aa26 | 1211 | |
e9174d1e SL |
1212 | macro_rules! require_simd { |
1213 | ($ty: expr, $position: expr) => { | |
1214 | require!($ty.is_simd(), "expected SIMD {} type, found non-SIMD `{}`", $position, $ty) | |
dfeec247 | 1215 | }; |
e9174d1e SL |
1216 | } |
1217 | ||
2c00a5a8 | 1218 | let tcx = bx.tcx(); |
dfeec247 XL |
1219 | let sig = tcx |
1220 | .normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), &callee_ty.fn_sig(tcx)); | |
476ff2be | 1221 | let arg_tys = sig.inputs(); |
e9174d1e | 1222 | |
0731742a XL |
1223 | if name == "simd_select_bitmask" { |
1224 | let in_ty = arg_tys[0]; | |
e74abb32 | 1225 | let m_len = match in_ty.kind { |
0731742a XL |
1226 | // Note that this `.unwrap()` crashes for isize/usize, that's sort |
1227 | // of intentional as there's not currently a use case for that. | |
ba9703b0 XL |
1228 | ty::Int(i) => i.bit_width().unwrap(), |
1229 | ty::Uint(i) => i.bit_width().unwrap(), | |
0731742a XL |
1230 | _ => return_error!("`{}` is not an integral type", in_ty), |
1231 | }; | |
1232 | require_simd!(arg_tys[1], "argument"); | |
1233 | let v_len = arg_tys[1].simd_size(tcx); | |
dfeec247 XL |
1234 | require!( |
1235 | m_len == v_len, | |
1236 | "mismatched lengths: mask length `{}` != other vector length `{}`", | |
1237 | m_len, | |
1238 | v_len | |
0731742a XL |
1239 | ); |
1240 | let i1 = bx.type_i1(); | |
60c5eb7d | 1241 | let i1xn = bx.type_vector(i1, m_len); |
0731742a XL |
1242 | let m_i1s = bx.bitcast(args[0].immediate(), i1xn); |
1243 | return Ok(bx.select(m_i1s, args[1].immediate(), args[2].immediate())); | |
1244 | } | |
1245 | ||
1246 | // every intrinsic below takes a SIMD vector as its first argument | |
e9174d1e SL |
1247 | require_simd!(arg_tys[0], "input"); |
1248 | let in_ty = arg_tys[0]; | |
1249 | let in_elem = arg_tys[0].simd_type(tcx); | |
1250 | let in_len = arg_tys[0].simd_size(tcx); | |
1251 | ||
1252 | let comparison = match name { | |
8faf50e0 XL |
1253 | "simd_eq" => Some(hir::BinOpKind::Eq), |
1254 | "simd_ne" => Some(hir::BinOpKind::Ne), | |
1255 | "simd_lt" => Some(hir::BinOpKind::Lt), | |
1256 | "simd_le" => Some(hir::BinOpKind::Le), | |
1257 | "simd_gt" => Some(hir::BinOpKind::Gt), | |
1258 | "simd_ge" => Some(hir::BinOpKind::Ge), | |
dfeec247 | 1259 | _ => None, |
e9174d1e SL |
1260 | }; |
1261 | ||
1262 | if let Some(cmp_op) = comparison { | |
1263 | require_simd!(ret_ty, "return"); | |
1264 | ||
1265 | let out_len = ret_ty.simd_size(tcx); | |
dfeec247 XL |
1266 | require!( |
1267 | in_len == out_len, | |
1268 | "expected return type with length {} (same as input type `{}`), \ | |
e9174d1e | 1269 | found `{}` with length {}", |
dfeec247 XL |
1270 | in_len, |
1271 | in_ty, | |
1272 | ret_ty, | |
1273 | out_len | |
1274 | ); | |
1275 | require!( | |
1276 | bx.type_kind(bx.element_type(llret_ty)) == TypeKind::Integer, | |
1277 | "expected return type with integer elements, found `{}` with non-integer `{}`", | |
1278 | ret_ty, | |
1279 | ret_ty.simd_type(tcx) | |
1280 | ); | |
1281 | ||
1282 | return Ok(compare_simd_types( | |
1283 | bx, | |
1284 | args[0].immediate(), | |
1285 | args[1].immediate(), | |
1286 | in_elem, | |
1287 | llret_ty, | |
1288 | cmp_op, | |
1289 | )); | |
e9174d1e SL |
1290 | } |
1291 | ||
1292 | if name.starts_with("simd_shuffle") { | |
dfeec247 XL |
1293 | let n: u64 = name["simd_shuffle".len()..].parse().unwrap_or_else(|_| { |
1294 | span_bug!(span, "bad `simd_shuffle` instruction only caught in codegen?") | |
1295 | }); | |
e9174d1e SL |
1296 | |
1297 | require_simd!(ret_ty, "return"); | |
1298 | ||
1299 | let out_len = ret_ty.simd_size(tcx); | |
dfeec247 XL |
1300 | require!( |
1301 | out_len == n, | |
1302 | "expected return type of length {}, found `{}` with length {}", | |
1303 | n, | |
1304 | ret_ty, | |
1305 | out_len | |
1306 | ); | |
1307 | require!( | |
1308 | in_elem == ret_ty.simd_type(tcx), | |
1309 | "expected return element type `{}` (element of input `{}`), \ | |
e9174d1e | 1310 | found `{}` with element type `{}`", |
dfeec247 XL |
1311 | in_elem, |
1312 | in_ty, | |
1313 | ret_ty, | |
1314 | ret_ty.simd_type(tcx) | |
1315 | ); | |
e9174d1e | 1316 | |
60c5eb7d | 1317 | let total_len = u128::from(in_len) * 2; |
e9174d1e | 1318 | |
ff7c6d11 | 1319 | let vector = args[2].immediate(); |
e9174d1e SL |
1320 | |
1321 | let indices: Option<Vec<_>> = (0..n) | |
1322 | .map(|i| { | |
1323 | let arg_idx = i; | |
a1dfa0c6 XL |
1324 | let val = bx.const_get_elt(vector, i as u64); |
1325 | match bx.const_to_opt_u128(val, true) { | |
e9174d1e SL |
1326 | None => { |
1327 | emit_error!("shuffle index #{} is not a constant", arg_idx); | |
1328 | None | |
1329 | } | |
1330 | Some(idx) if idx >= total_len => { | |
dfeec247 XL |
1331 | emit_error!( |
1332 | "shuffle index #{} is out of bounds (limit {})", | |
1333 | arg_idx, | |
1334 | total_len | |
1335 | ); | |
e9174d1e SL |
1336 | None |
1337 | } | |
a1dfa0c6 | 1338 | Some(idx) => Some(bx.const_i32(idx as i32)), |
e9174d1e SL |
1339 | } |
1340 | }) | |
1341 | .collect(); | |
1342 | let indices = match indices { | |
1343 | Some(i) => i, | |
dfeec247 | 1344 | None => return Ok(bx.const_null(llret_ty)), |
e9174d1e SL |
1345 | }; |
1346 | ||
dfeec247 XL |
1347 | return Ok(bx.shuffle_vector( |
1348 | args[0].immediate(), | |
1349 | args[1].immediate(), | |
1350 | bx.const_vector(&indices), | |
1351 | )); | |
e9174d1e SL |
1352 | } |
1353 | ||
1354 | if name == "simd_insert" { | |
dfeec247 XL |
1355 | require!( |
1356 | in_elem == arg_tys[2], | |
1357 | "expected inserted type `{}` (element of input `{}`), found `{}`", | |
1358 | in_elem, | |
1359 | in_ty, | |
1360 | arg_tys[2] | |
1361 | ); | |
1362 | return Ok(bx.insert_element( | |
1363 | args[0].immediate(), | |
1364 | args[2].immediate(), | |
1365 | args[1].immediate(), | |
1366 | )); | |
e9174d1e SL |
1367 | } |
1368 | if name == "simd_extract" { | |
dfeec247 XL |
1369 | require!( |
1370 | ret_ty == in_elem, | |
1371 | "expected return type `{}` (element of input `{}`), found `{}`", | |
1372 | in_elem, | |
1373 | in_ty, | |
1374 | ret_ty | |
1375 | ); | |
1376 | return Ok(bx.extract_element(args[0].immediate(), args[1].immediate())); | |
e9174d1e SL |
1377 | } |
1378 | ||
0531ce1d XL |
1379 | if name == "simd_select" { |
1380 | let m_elem_ty = in_elem; | |
1381 | let m_len = in_len; | |
0731742a | 1382 | require_simd!(arg_tys[1], "argument"); |
0531ce1d | 1383 | let v_len = arg_tys[1].simd_size(tcx); |
dfeec247 XL |
1384 | require!( |
1385 | m_len == v_len, | |
1386 | "mismatched lengths: mask length `{}` != other vector length `{}`", | |
1387 | m_len, | |
1388 | v_len | |
0531ce1d | 1389 | ); |
e74abb32 | 1390 | match m_elem_ty.kind { |
dfeec247 XL |
1391 | ty::Int(_) => {} |
1392 | _ => return_error!("mask element type is `{}`, expected `i_`", m_elem_ty), | |
0531ce1d XL |
1393 | } |
1394 | // truncate the mask to a vector of i1s | |
a1dfa0c6 XL |
1395 | let i1 = bx.type_i1(); |
1396 | let i1xn = bx.type_vector(i1, m_len as u64); | |
0531ce1d XL |
1397 | let m_i1s = bx.trunc(args[0].immediate(), i1xn); |
1398 | return Ok(bx.select(m_i1s, args[1].immediate(), args[2].immediate())); | |
1399 | } | |
1400 | ||
9fa01778 XL |
1401 | if name == "simd_bitmask" { |
1402 | // The `fn simd_bitmask(vector) -> unsigned integer` intrinsic takes a | |
1403 | // vector mask and returns an unsigned integer containing the most | |
1404 | // significant bit (MSB) of each lane. | |
9fa01778 XL |
1405 | |
1406 | // If the vector has less than 8 lanes, an u8 is returned with zeroed | |
1407 | // trailing bits. | |
1408 | let expected_int_bits = in_len.max(8); | |
e74abb32 | 1409 | match ret_ty.kind { |
ba9703b0 | 1410 | ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => (), |
dfeec247 | 1411 | _ => return_error!("bitmask `{}`, expected `u{}`", ret_ty, expected_int_bits), |
9fa01778 XL |
1412 | } |
1413 | ||
1414 | // Integer vector <i{in_bitwidth} x in_len>: | |
e74abb32 | 1415 | let (i_xn, in_elem_bitwidth) = match in_elem.kind { |
ba9703b0 XL |
1416 | ty::Int(i) => { |
1417 | (args[0].immediate(), i.bit_width().unwrap_or(bx.data_layout().pointer_size.bits())) | |
1418 | } | |
1419 | ty::Uint(i) => { | |
1420 | (args[0].immediate(), i.bit_width().unwrap_or(bx.data_layout().pointer_size.bits())) | |
1421 | } | |
9fa01778 XL |
1422 | _ => return_error!( |
1423 | "vector argument `{}`'s element type `{}`, expected integer element type", | |
dfeec247 XL |
1424 | in_ty, |
1425 | in_elem | |
9fa01778 XL |
1426 | ), |
1427 | }; | |
1428 | ||
1429 | // Shift the MSB to the right by "in_elem_bitwidth - 1" into the first bit position. | |
dfeec247 XL |
1430 | let shift_indices = |
1431 | vec![ | |
ba9703b0 | 1432 | bx.cx.const_int(bx.type_ix(in_elem_bitwidth), (in_elem_bitwidth - 1) as _); |
dfeec247 XL |
1433 | in_len as _ |
1434 | ]; | |
9fa01778 XL |
1435 | let i_xn_msb = bx.lshr(i_xn, bx.const_vector(shift_indices.as_slice())); |
1436 | // Truncate vector to an <i1 x N> | |
ba9703b0 | 1437 | let i1xn = bx.trunc(i_xn_msb, bx.type_vector(bx.type_i1(), in_len)); |
9fa01778 | 1438 | // Bitcast <i1 x N> to iN: |
ba9703b0 | 1439 | let i_ = bx.bitcast(i1xn, bx.type_ix(in_len)); |
9fa01778 | 1440 | // Zero-extend iN to the bitmask type: |
ba9703b0 | 1441 | return Ok(bx.zext(i_, bx.type_ix(expected_int_bits))); |
9fa01778 XL |
1442 | } |
1443 | ||
b7449926 XL |
1444 | fn simd_simple_float_intrinsic( |
1445 | name: &str, | |
ba9703b0 XL |
1446 | in_elem: &::rustc_middle::ty::TyS<'_>, |
1447 | in_ty: &::rustc_middle::ty::TyS<'_>, | |
60c5eb7d | 1448 | in_len: u64, |
a1dfa0c6 | 1449 | bx: &mut Builder<'a, 'll, 'tcx>, |
b7449926 | 1450 | span: Span, |
a1dfa0c6 | 1451 | args: &[OperandRef<'tcx, &'ll Value>], |
b7449926 | 1452 | ) -> Result<&'ll Value, ()> { |
94b46f34 XL |
1453 | macro_rules! emit_error { |
1454 | ($msg: tt) => { | |
1455 | emit_error!($msg, ) | |
1456 | }; | |
1457 | ($msg: tt, $($fmt: tt)*) => { | |
1458 | span_invalid_monomorphization_error( | |
1459 | bx.sess(), span, | |
0bf4aa26 | 1460 | &format!(concat!("invalid monomorphization of `{}` intrinsic: ", $msg), |
94b46f34 XL |
1461 | name, $($fmt)*)); |
1462 | } | |
1463 | } | |
1464 | macro_rules! return_error { | |
1465 | ($($fmt: tt)*) => { | |
1466 | { | |
1467 | emit_error!($($fmt)*); | |
1468 | return Err(()); | |
1469 | } | |
1470 | } | |
1471 | } | |
e74abb32 | 1472 | let ety = match in_elem.kind { |
b7449926 | 1473 | ty::Float(f) if f.bit_width() == 32 => { |
94b46f34 XL |
1474 | if in_len < 2 || in_len > 16 { |
1475 | return_error!( | |
1476 | "unsupported floating-point vector `{}` with length `{}` \ | |
1477 | out-of-range [2, 16]", | |
dfeec247 XL |
1478 | in_ty, |
1479 | in_len | |
1480 | ); | |
94b46f34 XL |
1481 | } |
1482 | "f32" | |
dfeec247 | 1483 | } |
b7449926 | 1484 | ty::Float(f) if f.bit_width() == 64 => { |
94b46f34 | 1485 | if in_len < 2 || in_len > 8 { |
dfeec247 XL |
1486 | return_error!( |
1487 | "unsupported floating-point vector `{}` with length `{}` \ | |
94b46f34 | 1488 | out-of-range [2, 8]", |
dfeec247 XL |
1489 | in_ty, |
1490 | in_len | |
1491 | ); | |
94b46f34 XL |
1492 | } |
1493 | "f64" | |
dfeec247 | 1494 | } |
b7449926 | 1495 | ty::Float(f) => { |
dfeec247 XL |
1496 | return_error!( |
1497 | "unsupported element type `{}` of floating-point vector `{}`", | |
1498 | f.name_str(), | |
1499 | in_ty | |
1500 | ); | |
1501 | } | |
94b46f34 XL |
1502 | _ => { |
1503 | return_error!("`{}` is not a floating-point type", in_ty); | |
1504 | } | |
1505 | }; | |
1506 | ||
1507 | let llvm_name = &format!("llvm.{0}.v{1}{2}", name, in_len, ety); | |
a1dfa0c6 | 1508 | let intrinsic = bx.get_intrinsic(&llvm_name); |
dfeec247 XL |
1509 | let c = |
1510 | bx.call(intrinsic, &args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(), None); | |
94b46f34 | 1511 | unsafe { llvm::LLVMRustSetHasUnsafeAlgebra(c) }; |
0bf4aa26 | 1512 | Ok(c) |
94b46f34 XL |
1513 | } |
1514 | ||
0bf4aa26 XL |
1515 | match name { |
1516 | "simd_fsqrt" => { | |
1517 | return simd_simple_float_intrinsic("sqrt", in_elem, in_ty, in_len, bx, span, args); | |
1518 | } | |
1519 | "simd_fsin" => { | |
1520 | return simd_simple_float_intrinsic("sin", in_elem, in_ty, in_len, bx, span, args); | |
1521 | } | |
1522 | "simd_fcos" => { | |
1523 | return simd_simple_float_intrinsic("cos", in_elem, in_ty, in_len, bx, span, args); | |
1524 | } | |
1525 | "simd_fabs" => { | |
1526 | return simd_simple_float_intrinsic("fabs", in_elem, in_ty, in_len, bx, span, args); | |
1527 | } | |
1528 | "simd_floor" => { | |
1529 | return simd_simple_float_intrinsic("floor", in_elem, in_ty, in_len, bx, span, args); | |
1530 | } | |
1531 | "simd_ceil" => { | |
1532 | return simd_simple_float_intrinsic("ceil", in_elem, in_ty, in_len, bx, span, args); | |
1533 | } | |
1534 | "simd_fexp" => { | |
1535 | return simd_simple_float_intrinsic("exp", in_elem, in_ty, in_len, bx, span, args); | |
1536 | } | |
1537 | "simd_fexp2" => { | |
1538 | return simd_simple_float_intrinsic("exp2", in_elem, in_ty, in_len, bx, span, args); | |
1539 | } | |
1540 | "simd_flog10" => { | |
1541 | return simd_simple_float_intrinsic("log10", in_elem, in_ty, in_len, bx, span, args); | |
1542 | } | |
1543 | "simd_flog2" => { | |
1544 | return simd_simple_float_intrinsic("log2", in_elem, in_ty, in_len, bx, span, args); | |
1545 | } | |
1546 | "simd_flog" => { | |
1547 | return simd_simple_float_intrinsic("log", in_elem, in_ty, in_len, bx, span, args); | |
1548 | } | |
1549 | "simd_fpowi" => { | |
1550 | return simd_simple_float_intrinsic("powi", in_elem, in_ty, in_len, bx, span, args); | |
1551 | } | |
1552 | "simd_fpow" => { | |
1553 | return simd_simple_float_intrinsic("pow", in_elem, in_ty, in_len, bx, span, args); | |
1554 | } | |
1555 | "simd_fma" => { | |
1556 | return simd_simple_float_intrinsic("fma", in_elem, in_ty, in_len, bx, span, args); | |
1557 | } | |
1558 | _ => { /* fallthrough */ } | |
94b46f34 XL |
1559 | } |
1560 | ||
1561 | // FIXME: use: | |
1562 | // https://github.com/llvm-mirror/llvm/blob/master/include/llvm/IR/Function.h#L182 | |
1563 | // https://github.com/llvm-mirror/llvm/blob/master/include/llvm/IR/Intrinsics.h#L81 | |
60c5eb7d | 1564 | fn llvm_vector_str(elem_ty: Ty<'_>, vec_len: u64, no_pointers: usize) -> String { |
94b46f34 | 1565 | let p0s: String = "p0".repeat(no_pointers); |
e74abb32 | 1566 | match elem_ty.kind { |
b7449926 XL |
1567 | ty::Int(v) => format!("v{}{}i{}", vec_len, p0s, v.bit_width().unwrap()), |
1568 | ty::Uint(v) => format!("v{}{}i{}", vec_len, p0s, v.bit_width().unwrap()), | |
1569 | ty::Float(v) => format!("v{}{}f{}", vec_len, p0s, v.bit_width()), | |
94b46f34 XL |
1570 | _ => unreachable!(), |
1571 | } | |
1572 | } | |
1573 | ||
dfeec247 XL |
1574 | fn llvm_vector_ty( |
1575 | cx: &CodegenCx<'ll, '_>, | |
1576 | elem_ty: Ty<'_>, | |
1577 | vec_len: u64, | |
1578 | mut no_pointers: usize, | |
1579 | ) -> &'ll Type { | |
94b46f34 | 1580 | // FIXME: use cx.layout_of(ty).llvm_type() ? |
e74abb32 | 1581 | let mut elem_ty = match elem_ty.kind { |
dfeec247 XL |
1582 | ty::Int(v) => cx.type_int_from_ty(v), |
1583 | ty::Uint(v) => cx.type_uint_from_ty(v), | |
1584 | ty::Float(v) => cx.type_float_from_ty(v), | |
94b46f34 XL |
1585 | _ => unreachable!(), |
1586 | }; | |
1587 | while no_pointers > 0 { | |
a1dfa0c6 | 1588 | elem_ty = cx.type_ptr_to(elem_ty); |
94b46f34 XL |
1589 | no_pointers -= 1; |
1590 | } | |
60c5eb7d | 1591 | cx.type_vector(elem_ty, vec_len) |
94b46f34 XL |
1592 | } |
1593 | ||
0bf4aa26 | 1594 | if name == "simd_gather" { |
94b46f34 XL |
1595 | // simd_gather(values: <N x T>, pointers: <N x *_ T>, |
1596 | // mask: <N x i{M}>) -> <N x T> | |
1597 | // * N: number of elements in the input vectors | |
1598 | // * T: type of the element to load | |
1599 | // * M: any integer width is supported, will be truncated to i1 | |
1600 | ||
1601 | // All types must be simd vector types | |
1602 | require_simd!(in_ty, "first"); | |
1603 | require_simd!(arg_tys[1], "second"); | |
1604 | require_simd!(arg_tys[2], "third"); | |
1605 | require_simd!(ret_ty, "return"); | |
1606 | ||
1607 | // Of the same length: | |
dfeec247 XL |
1608 | require!( |
1609 | in_len == arg_tys[1].simd_size(tcx), | |
1610 | "expected {} argument with length {} (same as input type `{}`), \ | |
1611 | found `{}` with length {}", | |
1612 | "second", | |
1613 | in_len, | |
1614 | in_ty, | |
1615 | arg_tys[1], | |
1616 | arg_tys[1].simd_size(tcx) | |
1617 | ); | |
1618 | require!( | |
1619 | in_len == arg_tys[2].simd_size(tcx), | |
1620 | "expected {} argument with length {} (same as input type `{}`), \ | |
1621 | found `{}` with length {}", | |
1622 | "third", | |
1623 | in_len, | |
1624 | in_ty, | |
1625 | arg_tys[2], | |
1626 | arg_tys[2].simd_size(tcx) | |
1627 | ); | |
94b46f34 XL |
1628 | |
1629 | // The return type must match the first argument type | |
dfeec247 | 1630 | require!(ret_ty == in_ty, "expected return type `{}`, found `{}`", in_ty, ret_ty); |
94b46f34 XL |
1631 | |
1632 | // This counts how many pointers | |
dc9dc135 | 1633 | fn ptr_count(t: Ty<'_>) -> usize { |
e74abb32 | 1634 | match t.kind { |
b7449926 | 1635 | ty::RawPtr(p) => 1 + ptr_count(p.ty), |
94b46f34 XL |
1636 | _ => 0, |
1637 | } | |
1638 | } | |
1639 | ||
1640 | // Non-ptr type | |
dc9dc135 | 1641 | fn non_ptr(t: Ty<'_>) -> Ty<'_> { |
e74abb32 | 1642 | match t.kind { |
b7449926 | 1643 | ty::RawPtr(p) => non_ptr(p.ty), |
94b46f34 XL |
1644 | _ => t, |
1645 | } | |
1646 | } | |
1647 | ||
1648 | // The second argument must be a simd vector with an element type that's a pointer | |
1649 | // to the element type of the first argument | |
e74abb32 | 1650 | let (pointer_count, underlying_ty) = match arg_tys[1].simd_type(tcx).kind { |
dfeec247 XL |
1651 | ty::RawPtr(p) if p.ty == in_elem => { |
1652 | (ptr_count(arg_tys[1].simd_type(tcx)), non_ptr(arg_tys[1].simd_type(tcx))) | |
1653 | } | |
94b46f34 | 1654 | _ => { |
dfeec247 XL |
1655 | require!( |
1656 | false, | |
1657 | "expected element type `{}` of second argument `{}` \ | |
94b46f34 XL |
1658 | to be a pointer to the element type `{}` of the first \ |
1659 | argument `{}`, found `{}` != `*_ {}`", | |
dfeec247 XL |
1660 | arg_tys[1].simd_type(tcx), |
1661 | arg_tys[1], | |
1662 | in_elem, | |
1663 | in_ty, | |
1664 | arg_tys[1].simd_type(tcx), | |
1665 | in_elem | |
1666 | ); | |
94b46f34 XL |
1667 | unreachable!(); |
1668 | } | |
1669 | }; | |
1670 | assert!(pointer_count > 0); | |
0bf4aa26 | 1671 | assert_eq!(pointer_count - 1, ptr_count(arg_tys[0].simd_type(tcx))); |
94b46f34 XL |
1672 | assert_eq!(underlying_ty, non_ptr(arg_tys[0].simd_type(tcx))); |
1673 | ||
1674 | // The element type of the third argument must be a signed integer type of any width: | |
e74abb32 | 1675 | match arg_tys[2].simd_type(tcx).kind { |
b7449926 | 1676 | ty::Int(_) => (), |
94b46f34 | 1677 | _ => { |
dfeec247 XL |
1678 | require!( |
1679 | false, | |
1680 | "expected element type `{}` of third argument `{}` \ | |
94b46f34 | 1681 | to be a signed integer type", |
dfeec247 XL |
1682 | arg_tys[2].simd_type(tcx), |
1683 | arg_tys[2] | |
1684 | ); | |
94b46f34 XL |
1685 | } |
1686 | } | |
1687 | ||
1688 | // Alignment of T, must be a constant integer value: | |
a1dfa0c6 XL |
1689 | let alignment_ty = bx.type_i32(); |
1690 | let alignment = bx.const_i32(bx.align_of(in_elem).bytes() as i32); | |
94b46f34 XL |
1691 | |
1692 | // Truncate the mask vector to a vector of i1s: | |
1693 | let (mask, mask_ty) = { | |
a1dfa0c6 | 1694 | let i1 = bx.type_i1(); |
60c5eb7d | 1695 | let i1xn = bx.type_vector(i1, in_len); |
94b46f34 XL |
1696 | (bx.trunc(args[2].immediate(), i1xn), i1xn) |
1697 | }; | |
1698 | ||
1699 | // Type of the vector of pointers: | |
a1dfa0c6 | 1700 | let llvm_pointer_vec_ty = llvm_vector_ty(bx, underlying_ty, in_len, pointer_count); |
94b46f34 XL |
1701 | let llvm_pointer_vec_str = llvm_vector_str(underlying_ty, in_len, pointer_count); |
1702 | ||
1703 | // Type of the vector of elements: | |
a1dfa0c6 | 1704 | let llvm_elem_vec_ty = llvm_vector_ty(bx, underlying_ty, in_len, pointer_count - 1); |
94b46f34 XL |
1705 | let llvm_elem_vec_str = llvm_vector_str(underlying_ty, in_len, pointer_count - 1); |
1706 | ||
dfeec247 XL |
1707 | let llvm_intrinsic = |
1708 | format!("llvm.masked.gather.{}.{}", llvm_elem_vec_str, llvm_pointer_vec_str); | |
1709 | let f = bx.declare_cfn( | |
1710 | &llvm_intrinsic, | |
1711 | bx.type_func( | |
1712 | &[llvm_pointer_vec_ty, alignment_ty, mask_ty, llvm_elem_vec_ty], | |
1713 | llvm_elem_vec_ty, | |
1714 | ), | |
1715 | ); | |
ba9703b0 | 1716 | llvm::SetUnnamedAddress(f, llvm::UnnamedAddr::No); |
dfeec247 | 1717 | let v = bx.call(f, &[args[1].immediate(), alignment, mask, args[0].immediate()], None); |
94b46f34 XL |
1718 | return Ok(v); |
1719 | } | |
1720 | ||
0bf4aa26 | 1721 | if name == "simd_scatter" { |
94b46f34 XL |
1722 | // simd_scatter(values: <N x T>, pointers: <N x *mut T>, |
1723 | // mask: <N x i{M}>) -> () | |
1724 | // * N: number of elements in the input vectors | |
1725 | // * T: type of the element to load | |
1726 | // * M: any integer width is supported, will be truncated to i1 | |
1727 | ||
1728 | // All types must be simd vector types | |
1729 | require_simd!(in_ty, "first"); | |
1730 | require_simd!(arg_tys[1], "second"); | |
1731 | require_simd!(arg_tys[2], "third"); | |
1732 | ||
1733 | // Of the same length: | |
dfeec247 XL |
1734 | require!( |
1735 | in_len == arg_tys[1].simd_size(tcx), | |
1736 | "expected {} argument with length {} (same as input type `{}`), \ | |
1737 | found `{}` with length {}", | |
1738 | "second", | |
1739 | in_len, | |
1740 | in_ty, | |
1741 | arg_tys[1], | |
1742 | arg_tys[1].simd_size(tcx) | |
1743 | ); | |
1744 | require!( | |
1745 | in_len == arg_tys[2].simd_size(tcx), | |
1746 | "expected {} argument with length {} (same as input type `{}`), \ | |
1747 | found `{}` with length {}", | |
1748 | "third", | |
1749 | in_len, | |
1750 | in_ty, | |
1751 | arg_tys[2], | |
1752 | arg_tys[2].simd_size(tcx) | |
1753 | ); | |
94b46f34 XL |
1754 | |
1755 | // This counts how many pointers | |
dc9dc135 | 1756 | fn ptr_count(t: Ty<'_>) -> usize { |
e74abb32 | 1757 | match t.kind { |
b7449926 | 1758 | ty::RawPtr(p) => 1 + ptr_count(p.ty), |
94b46f34 XL |
1759 | _ => 0, |
1760 | } | |
1761 | } | |
1762 | ||
1763 | // Non-ptr type | |
dc9dc135 | 1764 | fn non_ptr(t: Ty<'_>) -> Ty<'_> { |
e74abb32 | 1765 | match t.kind { |
b7449926 | 1766 | ty::RawPtr(p) => non_ptr(p.ty), |
94b46f34 XL |
1767 | _ => t, |
1768 | } | |
1769 | } | |
1770 | ||
1771 | // The second argument must be a simd vector with an element type that's a pointer | |
1772 | // to the element type of the first argument | |
e74abb32 | 1773 | let (pointer_count, underlying_ty) = match arg_tys[1].simd_type(tcx).kind { |
dfeec247 XL |
1774 | ty::RawPtr(p) if p.ty == in_elem && p.mutbl == hir::Mutability::Mut => { |
1775 | (ptr_count(arg_tys[1].simd_type(tcx)), non_ptr(arg_tys[1].simd_type(tcx))) | |
1776 | } | |
94b46f34 | 1777 | _ => { |
dfeec247 XL |
1778 | require!( |
1779 | false, | |
1780 | "expected element type `{}` of second argument `{}` \ | |
94b46f34 XL |
1781 | to be a pointer to the element type `{}` of the first \ |
1782 | argument `{}`, found `{}` != `*mut {}`", | |
dfeec247 XL |
1783 | arg_tys[1].simd_type(tcx), |
1784 | arg_tys[1], | |
1785 | in_elem, | |
1786 | in_ty, | |
1787 | arg_tys[1].simd_type(tcx), | |
1788 | in_elem | |
1789 | ); | |
94b46f34 XL |
1790 | unreachable!(); |
1791 | } | |
1792 | }; | |
1793 | assert!(pointer_count > 0); | |
0bf4aa26 | 1794 | assert_eq!(pointer_count - 1, ptr_count(arg_tys[0].simd_type(tcx))); |
94b46f34 XL |
1795 | assert_eq!(underlying_ty, non_ptr(arg_tys[0].simd_type(tcx))); |
1796 | ||
1797 | // The element type of the third argument must be a signed integer type of any width: | |
e74abb32 | 1798 | match arg_tys[2].simd_type(tcx).kind { |
b7449926 | 1799 | ty::Int(_) => (), |
94b46f34 | 1800 | _ => { |
dfeec247 XL |
1801 | require!( |
1802 | false, | |
1803 | "expected element type `{}` of third argument `{}` \ | |
94b46f34 | 1804 | to be a signed integer type", |
dfeec247 XL |
1805 | arg_tys[2].simd_type(tcx), |
1806 | arg_tys[2] | |
1807 | ); | |
94b46f34 XL |
1808 | } |
1809 | } | |
1810 | ||
1811 | // Alignment of T, must be a constant integer value: | |
a1dfa0c6 XL |
1812 | let alignment_ty = bx.type_i32(); |
1813 | let alignment = bx.const_i32(bx.align_of(in_elem).bytes() as i32); | |
94b46f34 XL |
1814 | |
1815 | // Truncate the mask vector to a vector of i1s: | |
1816 | let (mask, mask_ty) = { | |
a1dfa0c6 | 1817 | let i1 = bx.type_i1(); |
60c5eb7d | 1818 | let i1xn = bx.type_vector(i1, in_len); |
94b46f34 XL |
1819 | (bx.trunc(args[2].immediate(), i1xn), i1xn) |
1820 | }; | |
1821 | ||
a1dfa0c6 | 1822 | let ret_t = bx.type_void(); |
94b46f34 XL |
1823 | |
1824 | // Type of the vector of pointers: | |
a1dfa0c6 | 1825 | let llvm_pointer_vec_ty = llvm_vector_ty(bx, underlying_ty, in_len, pointer_count); |
94b46f34 XL |
1826 | let llvm_pointer_vec_str = llvm_vector_str(underlying_ty, in_len, pointer_count); |
1827 | ||
1828 | // Type of the vector of elements: | |
a1dfa0c6 | 1829 | let llvm_elem_vec_ty = llvm_vector_ty(bx, underlying_ty, in_len, pointer_count - 1); |
94b46f34 XL |
1830 | let llvm_elem_vec_str = llvm_vector_str(underlying_ty, in_len, pointer_count - 1); |
1831 | ||
dfeec247 XL |
1832 | let llvm_intrinsic = |
1833 | format!("llvm.masked.scatter.{}.{}", llvm_elem_vec_str, llvm_pointer_vec_str); | |
1834 | let f = bx.declare_cfn( | |
1835 | &llvm_intrinsic, | |
1836 | bx.type_func(&[llvm_elem_vec_ty, llvm_pointer_vec_ty, alignment_ty, mask_ty], ret_t), | |
1837 | ); | |
ba9703b0 | 1838 | llvm::SetUnnamedAddress(f, llvm::UnnamedAddr::No); |
dfeec247 | 1839 | let v = bx.call(f, &[args[0].immediate(), args[1].immediate(), alignment, mask], None); |
94b46f34 XL |
1840 | return Ok(v); |
1841 | } | |
1842 | ||
0531ce1d XL |
1843 | macro_rules! arith_red { |
1844 | ($name:tt : $integer_reduce:ident, $float_reduce:ident, $ordered:expr) => { | |
1845 | if name == $name { | |
dfeec247 XL |
1846 | require!( |
1847 | ret_ty == in_elem, | |
1848 | "expected return type `{}` (element of input `{}`), found `{}`", | |
1849 | in_elem, | |
1850 | in_ty, | |
1851 | ret_ty | |
1852 | ); | |
e74abb32 | 1853 | return match in_elem.kind { |
b7449926 | 1854 | ty::Int(_) | ty::Uint(_) => { |
0531ce1d XL |
1855 | let r = bx.$integer_reduce(args[0].immediate()); |
1856 | if $ordered { | |
1857 | // if overflow occurs, the result is the | |
1858 | // mathematical result modulo 2^n: | |
1859 | if name.contains("mul") { | |
1860 | Ok(bx.mul(args[1].immediate(), r)) | |
1861 | } else { | |
1862 | Ok(bx.add(args[1].immediate(), r)) | |
1863 | } | |
1864 | } else { | |
1865 | Ok(bx.$integer_reduce(args[0].immediate())) | |
1866 | } | |
dfeec247 | 1867 | } |
b7449926 | 1868 | ty::Float(f) => { |
0531ce1d | 1869 | let acc = if $ordered { |
416331ca XL |
1870 | // ordered arithmetic reductions take an accumulator |
1871 | args[1].immediate() | |
0531ce1d | 1872 | } else { |
416331ca XL |
1873 | // unordered arithmetic reductions use the identity accumulator |
1874 | let identity_acc = if $name.contains("mul") { 1.0 } else { 0.0 }; | |
0531ce1d | 1875 | match f.bit_width() { |
416331ca XL |
1876 | 32 => bx.const_real(bx.type_f32(), identity_acc), |
1877 | 64 => bx.const_real(bx.type_f64(), identity_acc), | |
dfeec247 XL |
1878 | v => return_error!( |
1879 | r#" | |
0531ce1d | 1880 | unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, |
dfeec247 XL |
1881 | $name, |
1882 | in_ty, | |
1883 | in_elem, | |
1884 | v, | |
1885 | ret_ty | |
1886 | ), | |
0531ce1d | 1887 | } |
0531ce1d XL |
1888 | }; |
1889 | Ok(bx.$float_reduce(acc, args[0].immediate())) | |
1890 | } | |
dfeec247 XL |
1891 | _ => return_error!( |
1892 | "unsupported {} from `{}` with element `{}` to `{}`", | |
1893 | $name, | |
1894 | in_ty, | |
1895 | in_elem, | |
1896 | ret_ty | |
1897 | ), | |
1898 | }; | |
0531ce1d | 1899 | } |
dfeec247 | 1900 | }; |
0531ce1d XL |
1901 | } |
1902 | ||
416331ca XL |
1903 | arith_red!("simd_reduce_add_ordered": vector_reduce_add, vector_reduce_fadd, true); |
1904 | arith_red!("simd_reduce_mul_ordered": vector_reduce_mul, vector_reduce_fmul, true); | |
0531ce1d XL |
1905 | arith_red!("simd_reduce_add_unordered": vector_reduce_add, vector_reduce_fadd_fast, false); |
1906 | arith_red!("simd_reduce_mul_unordered": vector_reduce_mul, vector_reduce_fmul_fast, false); | |
1907 | ||
1908 | macro_rules! minmax_red { | |
1909 | ($name:tt: $int_red:ident, $float_red:ident) => { | |
1910 | if name == $name { | |
dfeec247 XL |
1911 | require!( |
1912 | ret_ty == in_elem, | |
1913 | "expected return type `{}` (element of input `{}`), found `{}`", | |
1914 | in_elem, | |
1915 | in_ty, | |
1916 | ret_ty | |
1917 | ); | |
e74abb32 | 1918 | return match in_elem.kind { |
dfeec247 XL |
1919 | ty::Int(_i) => Ok(bx.$int_red(args[0].immediate(), true)), |
1920 | ty::Uint(_u) => Ok(bx.$int_red(args[0].immediate(), false)), | |
1921 | ty::Float(_f) => Ok(bx.$float_red(args[0].immediate())), | |
1922 | _ => return_error!( | |
1923 | "unsupported {} from `{}` with element `{}` to `{}`", | |
1924 | $name, | |
1925 | in_ty, | |
1926 | in_elem, | |
1927 | ret_ty | |
1928 | ), | |
1929 | }; | |
0531ce1d | 1930 | } |
dfeec247 | 1931 | }; |
0531ce1d XL |
1932 | } |
1933 | ||
1934 | minmax_red!("simd_reduce_min": vector_reduce_min, vector_reduce_fmin); | |
1935 | minmax_red!("simd_reduce_max": vector_reduce_max, vector_reduce_fmax); | |
1936 | ||
1937 | minmax_red!("simd_reduce_min_nanless": vector_reduce_min, vector_reduce_fmin_fast); | |
1938 | minmax_red!("simd_reduce_max_nanless": vector_reduce_max, vector_reduce_fmax_fast); | |
1939 | ||
1940 | macro_rules! bitwise_red { | |
1941 | ($name:tt : $red:ident, $boolean:expr) => { | |
1942 | if name == $name { | |
1943 | let input = if !$boolean { | |
dfeec247 XL |
1944 | require!( |
1945 | ret_ty == in_elem, | |
1946 | "expected return type `{}` (element of input `{}`), found `{}`", | |
1947 | in_elem, | |
1948 | in_ty, | |
1949 | ret_ty | |
1950 | ); | |
0531ce1d XL |
1951 | args[0].immediate() |
1952 | } else { | |
e74abb32 | 1953 | match in_elem.kind { |
dfeec247 XL |
1954 | ty::Int(_) | ty::Uint(_) => {} |
1955 | _ => return_error!( | |
1956 | "unsupported {} from `{}` with element `{}` to `{}`", | |
1957 | $name, | |
1958 | in_ty, | |
1959 | in_elem, | |
1960 | ret_ty | |
1961 | ), | |
0531ce1d XL |
1962 | } |
1963 | ||
1964 | // boolean reductions operate on vectors of i1s: | |
a1dfa0c6 XL |
1965 | let i1 = bx.type_i1(); |
1966 | let i1xn = bx.type_vector(i1, in_len as u64); | |
0531ce1d XL |
1967 | bx.trunc(args[0].immediate(), i1xn) |
1968 | }; | |
e74abb32 | 1969 | return match in_elem.kind { |
b7449926 | 1970 | ty::Int(_) | ty::Uint(_) => { |
0531ce1d | 1971 | let r = bx.$red(input); |
dfeec247 XL |
1972 | Ok(if !$boolean { r } else { bx.zext(r, bx.type_bool()) }) |
1973 | } | |
1974 | _ => return_error!( | |
1975 | "unsupported {} from `{}` with element `{}` to `{}`", | |
1976 | $name, | |
1977 | in_ty, | |
1978 | in_elem, | |
1979 | ret_ty | |
1980 | ), | |
1981 | }; | |
0531ce1d | 1982 | } |
dfeec247 | 1983 | }; |
0531ce1d XL |
1984 | } |
1985 | ||
1986 | bitwise_red!("simd_reduce_and": vector_reduce_and, false); | |
1987 | bitwise_red!("simd_reduce_or": vector_reduce_or, false); | |
1988 | bitwise_red!("simd_reduce_xor": vector_reduce_xor, false); | |
1989 | bitwise_red!("simd_reduce_all": vector_reduce_and, true); | |
1990 | bitwise_red!("simd_reduce_any": vector_reduce_or, true); | |
1991 | ||
e9174d1e SL |
1992 | if name == "simd_cast" { |
1993 | require_simd!(ret_ty, "return"); | |
1994 | let out_len = ret_ty.simd_size(tcx); | |
dfeec247 XL |
1995 | require!( |
1996 | in_len == out_len, | |
1997 | "expected return type with length {} (same as input type `{}`), \ | |
e9174d1e | 1998 | found `{}` with length {}", |
dfeec247 XL |
1999 | in_len, |
2000 | in_ty, | |
2001 | ret_ty, | |
2002 | out_len | |
2003 | ); | |
e9174d1e SL |
2004 | // casting cares about nominal type, not just structural type |
2005 | let out_elem = ret_ty.simd_type(tcx); | |
2006 | ||
dfeec247 XL |
2007 | if in_elem == out_elem { |
2008 | return Ok(args[0].immediate()); | |
2009 | } | |
e9174d1e | 2010 | |
dfeec247 XL |
2011 | enum Style { |
2012 | Float, | |
2013 | Int(/* is signed? */ bool), | |
2014 | Unsupported, | |
2015 | } | |
e9174d1e | 2016 | |
e74abb32 | 2017 | let (in_style, in_width) = match in_elem.kind { |
e9174d1e SL |
2018 | // vectors of pointer-sized integers should've been |
2019 | // disallowed before here, so this unwrap is safe. | |
b7449926 XL |
2020 | ty::Int(i) => (Style::Int(true), i.bit_width().unwrap()), |
2021 | ty::Uint(u) => (Style::Int(false), u.bit_width().unwrap()), | |
2022 | ty::Float(f) => (Style::Float, f.bit_width()), | |
dfeec247 | 2023 | _ => (Style::Unsupported, 0), |
e9174d1e | 2024 | }; |
e74abb32 | 2025 | let (out_style, out_width) = match out_elem.kind { |
b7449926 XL |
2026 | ty::Int(i) => (Style::Int(true), i.bit_width().unwrap()), |
2027 | ty::Uint(u) => (Style::Int(false), u.bit_width().unwrap()), | |
2028 | ty::Float(f) => (Style::Float, f.bit_width()), | |
dfeec247 | 2029 | _ => (Style::Unsupported, 0), |
e9174d1e SL |
2030 | }; |
2031 | ||
2032 | match (in_style, out_style) { | |
2033 | (Style::Int(in_is_signed), Style::Int(_)) => { | |
ff7c6d11 | 2034 | return Ok(match in_width.cmp(&out_width) { |
2c00a5a8 | 2035 | Ordering::Greater => bx.trunc(args[0].immediate(), llret_ty), |
ff7c6d11 | 2036 | Ordering::Equal => args[0].immediate(), |
dfeec247 XL |
2037 | Ordering::Less => { |
2038 | if in_is_signed { | |
2039 | bx.sext(args[0].immediate(), llret_ty) | |
2040 | } else { | |
2041 | bx.zext(args[0].immediate(), llret_ty) | |
2042 | } | |
e9174d1e | 2043 | } |
dfeec247 | 2044 | }); |
e9174d1e SL |
2045 | } |
2046 | (Style::Int(in_is_signed), Style::Float) => { | |
ff7c6d11 | 2047 | return Ok(if in_is_signed { |
2c00a5a8 | 2048 | bx.sitofp(args[0].immediate(), llret_ty) |
e9174d1e | 2049 | } else { |
2c00a5a8 | 2050 | bx.uitofp(args[0].immediate(), llret_ty) |
dfeec247 | 2051 | }); |
e9174d1e SL |
2052 | } |
2053 | (Style::Float, Style::Int(out_is_signed)) => { | |
ff7c6d11 | 2054 | return Ok(if out_is_signed { |
2c00a5a8 | 2055 | bx.fptosi(args[0].immediate(), llret_ty) |
e9174d1e | 2056 | } else { |
2c00a5a8 | 2057 | bx.fptoui(args[0].immediate(), llret_ty) |
dfeec247 | 2058 | }); |
e9174d1e SL |
2059 | } |
2060 | (Style::Float, Style::Float) => { | |
ff7c6d11 | 2061 | return Ok(match in_width.cmp(&out_width) { |
2c00a5a8 | 2062 | Ordering::Greater => bx.fptrunc(args[0].immediate(), llret_ty), |
ff7c6d11 | 2063 | Ordering::Equal => args[0].immediate(), |
dfeec247 XL |
2064 | Ordering::Less => bx.fpext(args[0].immediate(), llret_ty), |
2065 | }); | |
e9174d1e | 2066 | } |
dfeec247 | 2067 | _ => { /* Unsupported. Fallthrough. */ } |
e9174d1e | 2068 | } |
dfeec247 XL |
2069 | require!( |
2070 | false, | |
2071 | "unsupported cast from `{}` with element `{}` to `{}` with element `{}`", | |
2072 | in_ty, | |
2073 | in_elem, | |
2074 | ret_ty, | |
2075 | out_elem | |
2076 | ); | |
e9174d1e SL |
2077 | } |
2078 | macro_rules! arith { | |
32a655c1 | 2079 | ($($name: ident: $($($p: ident),* => $call: ident),*;)*) => { |
ff7c6d11 | 2080 | $(if name == stringify!($name) { |
e74abb32 | 2081 | match in_elem.kind { |
ff7c6d11 | 2082 | $($(ty::$p(_))|* => { |
2c00a5a8 | 2083 | return Ok(bx.$call(args[0].immediate(), args[1].immediate())) |
ff7c6d11 XL |
2084 | })* |
2085 | _ => {}, | |
2086 | } | |
2087 | require!(false, | |
0bf4aa26 XL |
2088 | "unsupported operation on `{}` with element `{}`", |
2089 | in_ty, | |
2090 | in_elem) | |
ff7c6d11 | 2091 | })* |
e9174d1e SL |
2092 | } |
2093 | } | |
2094 | arith! { | |
b7449926 XL |
2095 | simd_add: Uint, Int => add, Float => fadd; |
2096 | simd_sub: Uint, Int => sub, Float => fsub; | |
2097 | simd_mul: Uint, Int => mul, Float => fmul; | |
2098 | simd_div: Uint => udiv, Int => sdiv, Float => fdiv; | |
2099 | simd_rem: Uint => urem, Int => srem, Float => frem; | |
2100 | simd_shl: Uint, Int => shl; | |
2101 | simd_shr: Uint => lshr, Int => ashr; | |
2102 | simd_and: Uint, Int => and; | |
2103 | simd_or: Uint, Int => or; | |
2104 | simd_xor: Uint, Int => xor; | |
2105 | simd_fmax: Float => maxnum; | |
2106 | simd_fmin: Float => minnum; | |
9fa01778 | 2107 | |
e9174d1e | 2108 | } |
9fa01778 XL |
2109 | |
2110 | if name == "simd_saturating_add" || name == "simd_saturating_sub" { | |
2111 | let lhs = args[0].immediate(); | |
2112 | let rhs = args[1].immediate(); | |
2113 | let is_add = name == "simd_saturating_add"; | |
2114 | let ptr_bits = bx.tcx().data_layout.pointer_size.bits() as _; | |
e74abb32 | 2115 | let (signed, elem_width, elem_ty) = match in_elem.kind { |
dfeec247 XL |
2116 | ty::Int(i) => (true, i.bit_width().unwrap_or(ptr_bits), bx.cx.type_int_from_ty(i)), |
2117 | ty::Uint(i) => (false, i.bit_width().unwrap_or(ptr_bits), bx.cx.type_uint_from_ty(i)), | |
9fa01778 XL |
2118 | _ => { |
2119 | return_error!( | |
2120 | "expected element type `{}` of vector type `{}` \ | |
2121 | to be a signed or unsigned integer type", | |
dfeec247 XL |
2122 | arg_tys[0].simd_type(tcx), |
2123 | arg_tys[0] | |
9fa01778 XL |
2124 | ); |
2125 | } | |
2126 | }; | |
2127 | let llvm_intrinsic = &format!( | |
2128 | "llvm.{}{}.sat.v{}i{}", | |
2129 | if signed { 's' } else { 'u' }, | |
2130 | if is_add { "add" } else { "sub" }, | |
dfeec247 XL |
2131 | in_len, |
2132 | elem_width | |
9fa01778 XL |
2133 | ); |
2134 | let vec_ty = bx.cx.type_vector(elem_ty, in_len as u64); | |
2135 | ||
dfeec247 | 2136 | let f = bx.declare_cfn(&llvm_intrinsic, bx.type_func(&[vec_ty, vec_ty], vec_ty)); |
ba9703b0 | 2137 | llvm::SetUnnamedAddress(f, llvm::UnnamedAddr::No); |
9fa01778 XL |
2138 | let v = bx.call(f, &[lhs, rhs], None); |
2139 | return Ok(v); | |
2140 | } | |
2141 | ||
54a0048b | 2142 | span_bug!(span, "unknown SIMD intrinsic"); |
e9174d1e | 2143 | } |
92a42be0 | 2144 | |
ff7c6d11 | 2145 | // Returns the width of an int Ty, and if it's signed or not |
92a42be0 | 2146 | // Returns None if the type is not an integer |
32a655c1 SL |
2147 | // FIXME: there’s multiple of this functions, investigate using some of the already existing |
2148 | // stuffs. | |
9fa01778 | 2149 | fn int_type_width_signed(ty: Ty<'_>, cx: &CodegenCx<'_, '_>) -> Option<(u64, bool)> { |
e74abb32 | 2150 | match ty.kind { |
dfeec247 XL |
2151 | ty::Int(t) => Some(( |
2152 | match t { | |
ba9703b0 | 2153 | ast::IntTy::Isize => u64::from(cx.tcx.sess.target.ptr_width), |
dfeec247 XL |
2154 | ast::IntTy::I8 => 8, |
2155 | ast::IntTy::I16 => 16, | |
2156 | ast::IntTy::I32 => 32, | |
2157 | ast::IntTy::I64 => 64, | |
2158 | ast::IntTy::I128 => 128, | |
2159 | }, | |
2160 | true, | |
2161 | )), | |
2162 | ty::Uint(t) => Some(( | |
2163 | match t { | |
ba9703b0 | 2164 | ast::UintTy::Usize => u64::from(cx.tcx.sess.target.ptr_width), |
dfeec247 XL |
2165 | ast::UintTy::U8 => 8, |
2166 | ast::UintTy::U16 => 16, | |
2167 | ast::UintTy::U32 => 32, | |
2168 | ast::UintTy::U64 => 64, | |
2169 | ast::UintTy::U128 => 128, | |
2170 | }, | |
2171 | false, | |
2172 | )), | |
92a42be0 SL |
2173 | _ => None, |
2174 | } | |
2175 | } | |
54a0048b | 2176 | |
532ac7d7 | 2177 | // Returns the width of a float Ty |
54a0048b | 2178 | // Returns None if the type is not a float |
532ac7d7 | 2179 | fn float_type_width(ty: Ty<'_>) -> Option<u64> { |
e74abb32 | 2180 | match ty.kind { |
ba9703b0 | 2181 | ty::Float(t) => Some(t.bit_width()), |
54a0048b SL |
2182 | _ => None, |
2183 | } | |
2184 | } | |
f035d41b XL |
2185 | |
2186 | fn op_to_u32<'tcx>(op: &Operand<'tcx>) -> u32 { | |
2187 | Operand::scalar_from_const(op).to_u32().expect("Scalar is u32") | |
2188 | } |