1 use crate::abi
::{Abi, FnAbi, LlvmType, PassMode}
;
2 use crate::builder
::Builder
;
3 use crate::context
::CodegenCx
;
5 use crate::type_
::Type
;
6 use crate::type_of
::LayoutLlvmExt
;
7 use crate::va_arg
::emit_va_arg
;
8 use crate::value
::Value
;
10 use rustc_codegen_ssa
::base
::{compare_simd_types, wants_msvc_seh}
;
11 use rustc_codegen_ssa
::common
::span_invalid_monomorphization_error
;
12 use rustc_codegen_ssa
::common
::{IntPredicate, TypeKind}
;
13 use rustc_codegen_ssa
::mir
::operand
::OperandRef
;
14 use rustc_codegen_ssa
::mir
::place
::PlaceRef
;
15 use rustc_codegen_ssa
::traits
::*;
17 use rustc_middle
::ty
::layout
::{FnAbiExt, HasTyCtxt}
;
18 use rustc_middle
::ty
::{self, Ty}
;
19 use rustc_middle
::{bug, span_bug}
;
20 use rustc_span
::{sym, symbol::kw, Span, Symbol}
;
21 use rustc_target
::abi
::{self, HasDataLayout, LayoutOf, Primitive}
;
22 use rustc_target
::spec
::PanicStrategy
;
24 use std
::cmp
::Ordering
;
27 fn get_simple_intrinsic(cx
: &CodegenCx
<'ll
, '_
>, name
: Symbol
) -> Option
<&'ll Value
> {
28 let llvm_name
= match name
{
29 sym
::sqrtf32
=> "llvm.sqrt.f32",
30 sym
::sqrtf64
=> "llvm.sqrt.f64",
31 sym
::powif32
=> "llvm.powi.f32",
32 sym
::powif64
=> "llvm.powi.f64",
33 sym
::sinf32
=> "llvm.sin.f32",
34 sym
::sinf64
=> "llvm.sin.f64",
35 sym
::cosf32
=> "llvm.cos.f32",
36 sym
::cosf64
=> "llvm.cos.f64",
37 sym
::powf32
=> "llvm.pow.f32",
38 sym
::powf64
=> "llvm.pow.f64",
39 sym
::expf32
=> "llvm.exp.f32",
40 sym
::expf64
=> "llvm.exp.f64",
41 sym
::exp2f32
=> "llvm.exp2.f32",
42 sym
::exp2f64
=> "llvm.exp2.f64",
43 sym
::logf32
=> "llvm.log.f32",
44 sym
::logf64
=> "llvm.log.f64",
45 sym
::log10f32
=> "llvm.log10.f32",
46 sym
::log10f64
=> "llvm.log10.f64",
47 sym
::log2f32
=> "llvm.log2.f32",
48 sym
::log2f64
=> "llvm.log2.f64",
49 sym
::fmaf32
=> "llvm.fma.f32",
50 sym
::fmaf64
=> "llvm.fma.f64",
51 sym
::fabsf32
=> "llvm.fabs.f32",
52 sym
::fabsf64
=> "llvm.fabs.f64",
53 sym
::minnumf32
=> "llvm.minnum.f32",
54 sym
::minnumf64
=> "llvm.minnum.f64",
55 sym
::maxnumf32
=> "llvm.maxnum.f32",
56 sym
::maxnumf64
=> "llvm.maxnum.f64",
57 sym
::copysignf32
=> "llvm.copysign.f32",
58 sym
::copysignf64
=> "llvm.copysign.f64",
59 sym
::floorf32
=> "llvm.floor.f32",
60 sym
::floorf64
=> "llvm.floor.f64",
61 sym
::ceilf32
=> "llvm.ceil.f32",
62 sym
::ceilf64
=> "llvm.ceil.f64",
63 sym
::truncf32
=> "llvm.trunc.f32",
64 sym
::truncf64
=> "llvm.trunc.f64",
65 sym
::rintf32
=> "llvm.rint.f32",
66 sym
::rintf64
=> "llvm.rint.f64",
67 sym
::nearbyintf32
=> "llvm.nearbyint.f32",
68 sym
::nearbyintf64
=> "llvm.nearbyint.f64",
69 sym
::roundf32
=> "llvm.round.f32",
70 sym
::roundf64
=> "llvm.round.f64",
73 Some(cx
.get_intrinsic(&llvm_name
))
76 impl IntrinsicCallMethods
<'tcx
> for Builder
<'a
, 'll
, 'tcx
> {
77 fn codegen_intrinsic_call(
79 instance
: ty
::Instance
<'tcx
>,
80 fn_abi
: &FnAbi
<'tcx
, Ty
<'tcx
>>,
81 args
: &[OperandRef
<'tcx
, &'ll Value
>],
86 let callee_ty
= instance
.ty(tcx
, ty
::ParamEnv
::reveal_all());
88 let (def_id
, substs
) = match *callee_ty
.kind() {
89 ty
::FnDef(def_id
, substs
) => (def_id
, substs
),
90 _
=> bug
!("expected fn item type, found {}", callee_ty
),
93 let sig
= callee_ty
.fn_sig(tcx
);
94 let sig
= tcx
.normalize_erasing_late_bound_regions(ty
::ParamEnv
::reveal_all(), sig
);
95 let arg_tys
= sig
.inputs();
96 let ret_ty
= sig
.output();
97 let name
= tcx
.item_name(def_id
);
98 let name_str
= &*name
.as_str();
100 let llret_ty
= self.layout_of(ret_ty
).llvm_type(self);
101 let result
= PlaceRef
::new_sized(llresult
, fn_abi
.ret
.layout
);
103 let simple
= get_simple_intrinsic(self, name
);
104 let llval
= match name
{
105 _
if simple
.is_some() => self.call(
107 &args
.iter().map(|arg
| arg
.immediate()).collect
::<Vec
<_
>>(),
111 let expect
= self.get_intrinsic(&("llvm.expect.i1"));
112 self.call(expect
, &[args
[0].immediate(), self.const_bool(true)], None
)
115 let expect
= self.get_intrinsic(&("llvm.expect.i1"));
116 self.call(expect
, &[args
[0].immediate(), self.const_bool(false)], None
)
129 let llfn
= self.get_intrinsic(&("llvm.debugtrap"));
130 self.call(llfn
, &[], None
)
133 let intrinsic
= self.cx().get_intrinsic(&("llvm.va_copy"));
134 self.call(intrinsic
, &[args
[0].immediate(), args
[1].immediate()], None
)
137 match fn_abi
.ret
.layout
.abi
{
138 abi
::Abi
::Scalar(ref scalar
) => {
140 Primitive
::Int(..) => {
141 if self.cx().size_of(ret_ty
).bytes() < 4 {
142 // `va_arg` should not be called on a integer type
143 // less than 4 bytes in length. If it is, promote
144 // the integer to a `i32` and truncate the result
145 // back to the smaller type.
146 let promoted_result
= emit_va_arg(self, args
[0], tcx
.types
.i32);
147 self.trunc(promoted_result
, llret_ty
)
149 emit_va_arg(self, args
[0], ret_ty
)
152 Primitive
::F64
| Primitive
::Pointer
=> {
153 emit_va_arg(self, args
[0], ret_ty
)
155 // `va_arg` should never be used with the return type f32.
156 Primitive
::F32
=> bug
!("the va_arg intrinsic does not work with `f32`"),
159 _
=> bug
!("the va_arg intrinsic does not work with non-scalar types"),
163 sym
::volatile_load
| sym
::unaligned_volatile_load
=> {
164 let tp_ty
= substs
.type_at(0);
165 let ptr
= args
[0].immediate();
166 let load
= if let PassMode
::Cast(ty
) = fn_abi
.ret
.mode
{
167 let llty
= ty
.llvm_type(self);
168 let ptr
= self.pointercast(ptr
, self.type_ptr_to(llty
));
169 self.volatile_load(llty
, ptr
)
171 self.volatile_load(self.layout_of(tp_ty
).llvm_type(self), ptr
)
173 let align
= if name
== sym
::unaligned_volatile_load
{
176 self.align_of(tp_ty
).bytes() as u32
179 llvm
::LLVMSetAlignment(load
, align
);
181 self.to_immediate(load
, self.layout_of(tp_ty
))
183 sym
::volatile_store
=> {
184 let dst
= args
[0].deref(self.cx());
185 args
[1].val
.volatile_store(self, dst
);
188 sym
::unaligned_volatile_store
=> {
189 let dst
= args
[0].deref(self.cx());
190 args
[1].val
.unaligned_volatile_store(self, dst
);
193 sym
::prefetch_read_data
194 | sym
::prefetch_write_data
195 | sym
::prefetch_read_instruction
196 | sym
::prefetch_write_instruction
=> {
197 let expect
= self.get_intrinsic(&("llvm.prefetch"));
198 let (rw
, cache_type
) = match name
{
199 sym
::prefetch_read_data
=> (0, 1),
200 sym
::prefetch_write_data
=> (1, 1),
201 sym
::prefetch_read_instruction
=> (0, 0),
202 sym
::prefetch_write_instruction
=> (1, 0),
211 self.const_i32(cache_type
),
225 | sym
::saturating_add
226 | sym
::saturating_sub
=> {
228 match int_type_width_signed(ty
, self) {
229 Some((width
, signed
)) => match name
{
230 sym
::ctlz
| sym
::cttz
=> {
231 let y
= self.const_bool(false);
232 let llfn
= self.get_intrinsic(&format
!("llvm.{}.i{}", name
, width
));
233 self.call(llfn
, &[args
[0].immediate(), y
], None
)
235 sym
::ctlz_nonzero
| sym
::cttz_nonzero
=> {
236 let y
= self.const_bool(true);
237 let llvm_name
= &format
!("llvm.{}.i{}", &name_str
[..4], width
);
238 let llfn
= self.get_intrinsic(llvm_name
);
239 self.call(llfn
, &[args
[0].immediate(), y
], None
)
241 sym
::ctpop
=> self.call(
242 self.get_intrinsic(&format
!("llvm.ctpop.i{}", width
)),
243 &[args
[0].immediate()],
248 args
[0].immediate() // byte swap a u8/i8 is just a no-op
251 self.get_intrinsic(&format
!("llvm.bswap.i{}", width
)),
252 &[args
[0].immediate()],
257 sym
::bitreverse
=> self.call(
258 self.get_intrinsic(&format
!("llvm.bitreverse.i{}", width
)),
259 &[args
[0].immediate()],
262 sym
::rotate_left
| sym
::rotate_right
=> {
263 let is_left
= name
== sym
::rotate_left
;
264 let val
= args
[0].immediate();
265 let raw_shift
= args
[1].immediate();
266 // rotate = funnel shift with first two args the same
268 &format
!("llvm.fsh{}.i{}", if is_left { 'l' }
else { 'r' }
, width
);
269 let llfn
= self.get_intrinsic(llvm_name
);
270 self.call(llfn
, &[val
, val
, raw_shift
], None
)
272 sym
::saturating_add
| sym
::saturating_sub
=> {
273 let is_add
= name
== sym
::saturating_add
;
274 let lhs
= args
[0].immediate();
275 let rhs
= args
[1].immediate();
276 let llvm_name
= &format
!(
278 if signed { 's' }
else { 'u' }
,
279 if is_add { "add" }
else { "sub" }
,
282 let llfn
= self.get_intrinsic(llvm_name
);
283 self.call(llfn
, &[lhs
, rhs
], None
)
288 span_invalid_monomorphization_error(
292 "invalid monomorphization of `{}` intrinsic: \
293 expected basic integer type, found `{}`",
304 let tp_ty
= substs
.type_at(0);
305 let layout
= self.layout_of(tp_ty
).layout
;
306 let use_integer_compare
= match layout
.abi
{
307 Scalar(_
) | ScalarPair(_
, _
) => true,
308 Uninhabited
| Vector { .. }
=> false,
309 Aggregate { .. }
=> {
310 // For rusty ABIs, small aggregates are actually passed
311 // as `RegKind::Integer` (see `FnAbi::adjust_for_abi`),
312 // so we re-use that same threshold here.
313 layout
.size
<= self.data_layout().pointer_size
* 2
317 let a
= args
[0].immediate();
318 let b
= args
[1].immediate();
319 if layout
.size
.bytes() == 0 {
320 self.const_bool(true)
321 } else if use_integer_compare
{
322 let integer_ty
= self.type_ix(layout
.size
.bits());
323 let ptr_ty
= self.type_ptr_to(integer_ty
);
324 let a_ptr
= self.bitcast(a
, ptr_ty
);
325 let a_val
= self.load(integer_ty
, a_ptr
, layout
.align
.abi
);
326 let b_ptr
= self.bitcast(b
, ptr_ty
);
327 let b_val
= self.load(integer_ty
, b_ptr
, layout
.align
.abi
);
328 self.icmp(IntPredicate
::IntEQ
, a_val
, b_val
)
330 let i8p_ty
= self.type_i8p();
331 let a_ptr
= self.bitcast(a
, i8p_ty
);
332 let b_ptr
= self.bitcast(b
, i8p_ty
);
333 let n
= self.const_usize(layout
.size
.bytes());
334 let llfn
= self.get_intrinsic("memcmp");
335 let cmp
= self.call(llfn
, &[a_ptr
, b_ptr
, n
], None
);
336 self.icmp(IntPredicate
::IntEQ
, cmp
, self.const_i32(0))
340 _
if name_str
.starts_with("simd_") => {
341 match generic_simd_intrinsic(self, name
, callee_ty
, args
, ret_ty
, llret_ty
, span
) {
347 _
=> bug
!("unknown intrinsic '{}'", name
),
350 if !fn_abi
.ret
.is_ignore() {
351 if let PassMode
::Cast(ty
) = fn_abi
.ret
.mode
{
352 let ptr_llty
= self.type_ptr_to(ty
.llvm_type(self));
353 let ptr
= self.pointercast(result
.llval
, ptr_llty
);
354 self.store(llval
, ptr
, result
.align
);
356 OperandRef
::from_immediate_or_packed_pair(self, llval
, result
.layout
)
358 .store(self, result
);
363 fn abort(&mut self) {
364 let fnname
= self.get_intrinsic(&("llvm.trap"));
365 self.call(fnname
, &[], None
);
368 fn assume(&mut self, val
: Self::Value
) {
369 let assume_intrinsic
= self.get_intrinsic("llvm.assume");
370 self.call(assume_intrinsic
, &[val
], None
);
373 fn expect(&mut self, cond
: Self::Value
, expected
: bool
) -> Self::Value
{
374 let expect
= self.get_intrinsic(&"llvm.expect.i1");
375 self.call(expect
, &[cond
, self.const_bool(expected
)], None
)
378 fn sideeffect(&mut self) {
379 // This kind of check would make a ton of sense in the caller, but currently the only
380 // caller of this function is in `rustc_codegen_ssa`, which is agnostic to whether LLVM
381 // codegen backend being used, and so is unable to check the LLVM version.
382 if unsafe { llvm::LLVMRustVersionMajor() }
< 12 {
383 let fnname
= self.get_intrinsic(&("llvm.sideeffect"));
384 self.call(fnname
, &[], None
);
388 fn va_start(&mut self, va_list
: &'ll Value
) -> &'ll Value
{
389 let intrinsic
= self.cx().get_intrinsic("llvm.va_start");
390 self.call(intrinsic
, &[va_list
], None
)
393 fn va_end(&mut self, va_list
: &'ll Value
) -> &'ll Value
{
394 let intrinsic
= self.cx().get_intrinsic("llvm.va_end");
395 self.call(intrinsic
, &[va_list
], None
)
400 bx
: &mut Builder
<'a
, 'll
, 'tcx
>,
401 try_func
: &'ll Value
,
403 catch_func
: &'ll Value
,
406 if bx
.sess().panic_strategy() == PanicStrategy
::Abort
{
407 bx
.call(try_func
, &[data
], None
);
408 // Return 0 unconditionally from the intrinsic call;
409 // we can never unwind.
410 let ret_align
= bx
.tcx().data_layout
.i32_align
.abi
;
411 bx
.store(bx
.const_i32(0), dest
, ret_align
);
412 } else if wants_msvc_seh(bx
.sess()) {
413 codegen_msvc_try(bx
, try_func
, data
, catch_func
, dest
);
414 } else if bx
.sess().target
.is_like_emscripten
{
415 codegen_emcc_try(bx
, try_func
, data
, catch_func
, dest
);
417 codegen_gnu_try(bx
, try_func
, data
, catch_func
, dest
);
421 // MSVC's definition of the `rust_try` function.
423 // This implementation uses the new exception handling instructions in LLVM
424 // which have support in LLVM for SEH on MSVC targets. Although these
425 // instructions are meant to work for all targets, as of the time of this
426 // writing, however, LLVM does not recommend the usage of these new instructions
427 // as the old ones are still more optimized.
429 bx
: &mut Builder
<'a
, 'll
, 'tcx
>,
430 try_func
: &'ll Value
,
432 catch_func
: &'ll Value
,
435 let llfn
= get_rust_try_fn(bx
, &mut |mut bx
| {
436 bx
.set_personality_fn(bx
.eh_personality());
438 let mut normal
= bx
.build_sibling_block("normal");
439 let mut catchswitch
= bx
.build_sibling_block("catchswitch");
440 let mut catchpad_rust
= bx
.build_sibling_block("catchpad_rust");
441 let mut catchpad_foreign
= bx
.build_sibling_block("catchpad_foreign");
442 let mut caught
= bx
.build_sibling_block("caught");
444 let try_func
= llvm
::get_param(bx
.llfn(), 0);
445 let data
= llvm
::get_param(bx
.llfn(), 1);
446 let catch_func
= llvm
::get_param(bx
.llfn(), 2);
448 // We're generating an IR snippet that looks like:
450 // declare i32 @rust_try(%try_func, %data, %catch_func) {
451 // %slot = alloca i8*
452 // invoke %try_func(%data) to label %normal unwind label %catchswitch
458 // %cs = catchswitch within none [%catchpad_rust, %catchpad_foreign] unwind to caller
461 // %tok = catchpad within %cs [%type_descriptor, 8, %slot]
463 // call %catch_func(%data, %ptr)
464 // catchret from %tok to label %caught
467 // %tok = catchpad within %cs [null, 64, null]
468 // call %catch_func(%data, null)
469 // catchret from %tok to label %caught
475 // This structure follows the basic usage of throw/try/catch in LLVM.
476 // For example, compile this C++ snippet to see what LLVM generates:
478 // struct rust_panic {
479 // rust_panic(const rust_panic&);
486 // void (*try_func)(void*),
488 // void (*catch_func)(void*, void*) noexcept
493 // } catch(rust_panic& a) {
494 // catch_func(data, &a);
497 // catch_func(data, NULL);
502 // More information can be found in libstd's seh.rs implementation.
503 let ptr_align
= bx
.tcx().data_layout
.pointer_align
.abi
;
504 let slot
= bx
.alloca(bx
.type_i8p(), ptr_align
);
505 bx
.invoke(try_func
, &[data
], normal
.llbb(), catchswitch
.llbb(), None
);
507 normal
.ret(bx
.const_i32(0));
509 let cs
= catchswitch
.catch_switch(None
, None
, 2);
510 catchswitch
.add_handler(cs
, catchpad_rust
.llbb());
511 catchswitch
.add_handler(cs
, catchpad_foreign
.llbb());
513 // We can't use the TypeDescriptor defined in libpanic_unwind because it
514 // might be in another DLL and the SEH encoding only supports specifying
515 // a TypeDescriptor from the current module.
517 // However this isn't an issue since the MSVC runtime uses string
518 // comparison on the type name to match TypeDescriptors rather than
521 // So instead we generate a new TypeDescriptor in each module that uses
522 // `try` and let the linker merge duplicate definitions in the same
525 // When modifying, make sure that the type_name string exactly matches
526 // the one used in src/libpanic_unwind/seh.rs.
527 let type_info_vtable
= bx
.declare_global("??_7type_info@@6B@", bx
.type_i8p());
528 let type_name
= bx
.const_bytes(b
"rust_panic\0");
530 bx
.const_struct(&[type_info_vtable
, bx
.const_null(bx
.type_i8p()), type_name
], false);
531 let tydesc
= bx
.declare_global("__rust_panic_type_info", bx
.val_ty(type_info
));
533 llvm
::LLVMRustSetLinkage(tydesc
, llvm
::Linkage
::LinkOnceODRLinkage
);
534 llvm
::SetUniqueComdat(bx
.llmod
, tydesc
);
535 llvm
::LLVMSetInitializer(tydesc
, type_info
);
538 // The flag value of 8 indicates that we are catching the exception by
539 // reference instead of by value. We can't use catch by value because
540 // that requires copying the exception object, which we don't support
541 // since our exception object effectively contains a Box.
543 // Source: MicrosoftCXXABI::getAddrOfCXXCatchHandlerType in clang
544 let flags
= bx
.const_i32(8);
545 let funclet
= catchpad_rust
.catch_pad(cs
, &[tydesc
, flags
, slot
]);
546 let ptr
= catchpad_rust
.load(bx
.type_i8p(), slot
, ptr_align
);
547 catchpad_rust
.call(catch_func
, &[data
, ptr
], Some(&funclet
));
548 catchpad_rust
.catch_ret(&funclet
, caught
.llbb());
550 // The flag value of 64 indicates a "catch-all".
551 let flags
= bx
.const_i32(64);
552 let null
= bx
.const_null(bx
.type_i8p());
553 let funclet
= catchpad_foreign
.catch_pad(cs
, &[null
, flags
, null
]);
554 catchpad_foreign
.call(catch_func
, &[data
, null
], Some(&funclet
));
555 catchpad_foreign
.catch_ret(&funclet
, caught
.llbb());
557 caught
.ret(bx
.const_i32(1));
560 // Note that no invoke is used here because by definition this function
561 // can't panic (that's what it's catching).
562 let ret
= bx
.call(llfn
, &[try_func
, data
, catch_func
], None
);
563 let i32_align
= bx
.tcx().data_layout
.i32_align
.abi
;
564 bx
.store(ret
, dest
, i32_align
);
567 // Definition of the standard `try` function for Rust using the GNU-like model
568 // of exceptions (e.g., the normal semantics of LLVM's `landingpad` and `invoke`
571 // This codegen is a little surprising because we always call a shim
572 // function instead of inlining the call to `invoke` manually here. This is done
573 // because in LLVM we're only allowed to have one personality per function
574 // definition. The call to the `try` intrinsic is being inlined into the
575 // function calling it, and that function may already have other personality
576 // functions in play. By calling a shim we're guaranteed that our shim will have
577 // the right personality function.
579 bx
: &mut Builder
<'a
, 'll
, 'tcx
>,
580 try_func
: &'ll Value
,
582 catch_func
: &'ll Value
,
585 let llfn
= get_rust_try_fn(bx
, &mut |mut bx
| {
586 // Codegens the shims described above:
589 // invoke %try_func(%data) normal %normal unwind %catch
595 // (%ptr, _) = landingpad
596 // call %catch_func(%data, %ptr)
598 let mut then
= bx
.build_sibling_block("then");
599 let mut catch = bx
.build_sibling_block("catch");
601 let try_func
= llvm
::get_param(bx
.llfn(), 0);
602 let data
= llvm
::get_param(bx
.llfn(), 1);
603 let catch_func
= llvm
::get_param(bx
.llfn(), 2);
604 bx
.invoke(try_func
, &[data
], then
.llbb(), catch.llbb(), None
);
605 then
.ret(bx
.const_i32(0));
607 // Type indicator for the exception being thrown.
609 // The first value in this tuple is a pointer to the exception object
610 // being thrown. The second value is a "selector" indicating which of
611 // the landing pad clauses the exception's type had been matched to.
612 // rust_try ignores the selector.
613 let lpad_ty
= bx
.type_struct(&[bx
.type_i8p(), bx
.type_i32()], false);
614 let vals
= catch.landing_pad(lpad_ty
, bx
.eh_personality(), 1);
615 let tydesc
= bx
.const_null(bx
.type_i8p());
616 catch.add_clause(vals
, tydesc
);
617 let ptr
= catch.extract_value(vals
, 0);
618 catch.call(catch_func
, &[data
, ptr
], None
);
619 catch.ret(bx
.const_i32(1));
622 // Note that no invoke is used here because by definition this function
623 // can't panic (that's what it's catching).
624 let ret
= bx
.call(llfn
, &[try_func
, data
, catch_func
], None
);
625 let i32_align
= bx
.tcx().data_layout
.i32_align
.abi
;
626 bx
.store(ret
, dest
, i32_align
);
629 // Variant of codegen_gnu_try used for emscripten where Rust panics are
630 // implemented using C++ exceptions. Here we use exceptions of a specific type
631 // (`struct rust_panic`) to represent Rust panics.
633 bx
: &mut Builder
<'a
, 'll
, 'tcx
>,
634 try_func
: &'ll Value
,
636 catch_func
: &'ll Value
,
639 let llfn
= get_rust_try_fn(bx
, &mut |mut bx
| {
640 // Codegens the shims described above:
643 // invoke %try_func(%data) normal %normal unwind %catch
649 // (%ptr, %selector) = landingpad
650 // %rust_typeid = @llvm.eh.typeid.for(@_ZTI10rust_panic)
651 // %is_rust_panic = %selector == %rust_typeid
652 // %catch_data = alloca { i8*, i8 }
653 // %catch_data[0] = %ptr
654 // %catch_data[1] = %is_rust_panic
655 // call %catch_func(%data, %catch_data)
657 let mut then
= bx
.build_sibling_block("then");
658 let mut catch = bx
.build_sibling_block("catch");
660 let try_func
= llvm
::get_param(bx
.llfn(), 0);
661 let data
= llvm
::get_param(bx
.llfn(), 1);
662 let catch_func
= llvm
::get_param(bx
.llfn(), 2);
663 bx
.invoke(try_func
, &[data
], then
.llbb(), catch.llbb(), None
);
664 then
.ret(bx
.const_i32(0));
666 // Type indicator for the exception being thrown.
668 // The first value in this tuple is a pointer to the exception object
669 // being thrown. The second value is a "selector" indicating which of
670 // the landing pad clauses the exception's type had been matched to.
671 let tydesc
= bx
.eh_catch_typeinfo();
672 let lpad_ty
= bx
.type_struct(&[bx
.type_i8p(), bx
.type_i32()], false);
673 let vals
= catch.landing_pad(lpad_ty
, bx
.eh_personality(), 2);
674 catch.add_clause(vals
, tydesc
);
675 catch.add_clause(vals
, bx
.const_null(bx
.type_i8p()));
676 let ptr
= catch.extract_value(vals
, 0);
677 let selector
= catch.extract_value(vals
, 1);
679 // Check if the typeid we got is the one for a Rust panic.
680 let llvm_eh_typeid_for
= bx
.get_intrinsic("llvm.eh.typeid.for");
681 let rust_typeid
= catch.call(llvm_eh_typeid_for
, &[tydesc
], None
);
682 let is_rust_panic
= catch.icmp(IntPredicate
::IntEQ
, selector
, rust_typeid
);
683 let is_rust_panic
= catch.zext(is_rust_panic
, bx
.type_bool());
685 // We need to pass two values to catch_func (ptr and is_rust_panic), so
686 // create an alloca and pass a pointer to that.
687 let ptr_align
= bx
.tcx().data_layout
.pointer_align
.abi
;
688 let i8_align
= bx
.tcx().data_layout
.i8_align
.abi
;
690 catch.alloca(bx
.type_struct(&[bx
.type_i8p(), bx
.type_bool()], false), ptr_align
);
691 let catch_data_0
= catch.inbounds_gep(catch_data
, &[bx
.const_usize(0), bx
.const_usize(0)]);
692 catch.store(ptr
, catch_data_0
, ptr_align
);
693 let catch_data_1
= catch.inbounds_gep(catch_data
, &[bx
.const_usize(0), bx
.const_usize(1)]);
694 catch.store(is_rust_panic
, catch_data_1
, i8_align
);
695 let catch_data
= catch.bitcast(catch_data
, bx
.type_i8p());
697 catch.call(catch_func
, &[data
, catch_data
], None
);
698 catch.ret(bx
.const_i32(1));
701 // Note that no invoke is used here because by definition this function
702 // can't panic (that's what it's catching).
703 let ret
= bx
.call(llfn
, &[try_func
, data
, catch_func
], None
);
704 let i32_align
= bx
.tcx().data_layout
.i32_align
.abi
;
705 bx
.store(ret
, dest
, i32_align
);
708 // Helper function to give a Block to a closure to codegen a shim function.
709 // This is currently primarily used for the `try` intrinsic functions above.
710 fn gen_fn
<'ll
, 'tcx
>(
711 cx
: &CodegenCx
<'ll
, 'tcx
>,
713 rust_fn_sig
: ty
::PolyFnSig
<'tcx
>,
714 codegen
: &mut dyn FnMut(Builder
<'_
, 'll
, 'tcx
>),
716 let fn_abi
= FnAbi
::of_fn_ptr(cx
, rust_fn_sig
, &[]);
717 let llfn
= cx
.declare_fn(name
, &fn_abi
);
718 cx
.set_frame_pointer_type(llfn
);
719 cx
.apply_target_cpu_attr(llfn
);
720 // FIXME(eddyb) find a nicer way to do this.
721 unsafe { llvm::LLVMRustSetLinkage(llfn, llvm::Linkage::InternalLinkage) }
;
722 let llbb
= Builder
::append_block(cx
, llfn
, "entry-block");
723 let bx
= Builder
::build(cx
, llbb
);
728 // Helper function used to get a handle to the `__rust_try` function used to
731 // This function is only generated once and is then cached.
732 fn get_rust_try_fn
<'ll
, 'tcx
>(
733 cx
: &CodegenCx
<'ll
, 'tcx
>,
734 codegen
: &mut dyn FnMut(Builder
<'_
, 'll
, 'tcx
>),
736 if let Some(llfn
) = cx
.rust_try_fn
.get() {
740 // Define the type up front for the signature of the rust_try function.
742 let i8p
= tcx
.mk_mut_ptr(tcx
.types
.i8);
743 // `unsafe fn(*mut i8) -> ()`
744 let try_fn_ty
= tcx
.mk_fn_ptr(ty
::Binder
::dummy(tcx
.mk_fn_sig(
748 hir
::Unsafety
::Unsafe
,
751 // `unsafe fn(*mut i8, *mut i8) -> ()`
752 let catch_fn_ty
= tcx
.mk_fn_ptr(ty
::Binder
::dummy(tcx
.mk_fn_sig(
753 [i8p
, i8p
].iter().cloned(),
756 hir
::Unsafety
::Unsafe
,
759 // `unsafe fn(unsafe fn(*mut i8) -> (), *mut i8, unsafe fn(*mut i8, *mut i8) -> ()) -> i32`
760 let rust_fn_sig
= ty
::Binder
::dummy(cx
.tcx
.mk_fn_sig(
761 vec
![try_fn_ty
, i8p
, catch_fn_ty
].into_iter(),
764 hir
::Unsafety
::Unsafe
,
767 let rust_try
= gen_fn(cx
, "__rust_try", rust_fn_sig
, codegen
);
768 cx
.rust_try_fn
.set(Some(rust_try
));
772 fn generic_simd_intrinsic(
773 bx
: &mut Builder
<'a
, 'll
, 'tcx
>,
776 args
: &[OperandRef
<'tcx
, &'ll Value
>],
780 ) -> Result
<&'ll Value
, ()> {
781 // macros for error handling:
782 macro_rules
! emit_error
{
786 ($msg
: tt
, $
($fmt
: tt
)*) => {
787 span_invalid_monomorphization_error(
789 &format
!(concat
!("invalid monomorphization of `{}` intrinsic: ", $msg
),
794 macro_rules
! return_error
{
797 emit_error
!($
($fmt
)*);
803 macro_rules
! require
{
804 ($cond
: expr
, $
($fmt
: tt
)*) => {
806 return_error
!($
($fmt
)*);
811 macro_rules
! require_simd
{
812 ($ty
: expr
, $position
: expr
) => {
813 require
!($ty
.is_simd(), "expected SIMD {} type, found non-SIMD `{}`", $position
, $ty
)
819 tcx
.normalize_erasing_late_bound_regions(ty
::ParamEnv
::reveal_all(), callee_ty
.fn_sig(tcx
));
820 let arg_tys
= sig
.inputs();
821 let name_str
= &*name
.as_str();
823 if name
== sym
::simd_select_bitmask
{
824 let in_ty
= arg_tys
[0];
825 let m_len
= match in_ty
.kind() {
826 // Note that this `.unwrap()` crashes for isize/usize, that's sort
827 // of intentional as there's not currently a use case for that.
828 ty
::Int(i
) => i
.bit_width().unwrap(),
829 ty
::Uint(i
) => i
.bit_width().unwrap(),
830 _
=> return_error
!("`{}` is not an integral type", in_ty
),
832 require_simd
!(arg_tys
[1], "argument");
833 let (v_len
, _
) = arg_tys
[1].simd_size_and_type(bx
.tcx());
835 // Allow masks for vectors with fewer than 8 elements to be
836 // represented with a u8 or i8.
837 m_len
== v_len
|| (m_len
== 8 && v_len
< 8),
838 "mismatched lengths: mask length `{}` != other vector length `{}`",
842 let i1
= bx
.type_i1();
843 let im
= bx
.type_ix(v_len
);
844 let i1xn
= bx
.type_vector(i1
, v_len
);
845 let m_im
= bx
.trunc(args
[0].immediate(), im
);
846 let m_i1s
= bx
.bitcast(m_im
, i1xn
);
847 return Ok(bx
.select(m_i1s
, args
[1].immediate(), args
[2].immediate()));
850 // every intrinsic below takes a SIMD vector as its first argument
851 require_simd
!(arg_tys
[0], "input");
852 let in_ty
= arg_tys
[0];
854 let comparison
= match name
{
855 sym
::simd_eq
=> Some(hir
::BinOpKind
::Eq
),
856 sym
::simd_ne
=> Some(hir
::BinOpKind
::Ne
),
857 sym
::simd_lt
=> Some(hir
::BinOpKind
::Lt
),
858 sym
::simd_le
=> Some(hir
::BinOpKind
::Le
),
859 sym
::simd_gt
=> Some(hir
::BinOpKind
::Gt
),
860 sym
::simd_ge
=> Some(hir
::BinOpKind
::Ge
),
864 let (in_len
, in_elem
) = arg_tys
[0].simd_size_and_type(bx
.tcx());
865 if let Some(cmp_op
) = comparison
{
866 require_simd
!(ret_ty
, "return");
868 let (out_len
, out_ty
) = ret_ty
.simd_size_and_type(bx
.tcx());
871 "expected return type with length {} (same as input type `{}`), \
872 found `{}` with length {}",
879 bx
.type_kind(bx
.element_type(llret_ty
)) == TypeKind
::Integer
,
880 "expected return type with integer elements, found `{}` with non-integer `{}`",
885 return Ok(compare_simd_types(
895 if let Some(stripped
) = name_str
.strip_prefix("simd_shuffle") {
896 let n
: u64 = stripped
.parse().unwrap_or_else(|_
| {
897 span_bug
!(span
, "bad `simd_shuffle` instruction only caught in codegen?")
900 require_simd
!(ret_ty
, "return");
902 let (out_len
, out_ty
) = ret_ty
.simd_size_and_type(bx
.tcx());
905 "expected return type of length {}, found `{}` with length {}",
912 "expected return element type `{}` (element of input `{}`), \
913 found `{}` with element type `{}`",
920 let total_len
= u128
::from(in_len
) * 2;
922 let vector
= args
[2].immediate();
924 let indices
: Option
<Vec
<_
>> = (0..n
)
927 let val
= bx
.const_get_elt(vector
, i
as u64);
928 match bx
.const_to_opt_u128(val
, true) {
930 emit_error
!("shuffle index #{} is not a constant", arg_idx
);
933 Some(idx
) if idx
>= total_len
=> {
935 "shuffle index #{} is out of bounds (limit {})",
941 Some(idx
) => Some(bx
.const_i32(idx
as i32)),
945 let indices
= match indices
{
947 None
=> return Ok(bx
.const_null(llret_ty
)),
950 return Ok(bx
.shuffle_vector(
953 bx
.const_vector(&indices
),
957 if name
== sym
::simd_insert
{
959 in_elem
== arg_tys
[2],
960 "expected inserted type `{}` (element of input `{}`), found `{}`",
965 return Ok(bx
.insert_element(
971 if name
== sym
::simd_extract
{
974 "expected return type `{}` (element of input `{}`), found `{}`",
979 return Ok(bx
.extract_element(args
[0].immediate(), args
[1].immediate()));
982 if name
== sym
::simd_select
{
983 let m_elem_ty
= in_elem
;
985 require_simd
!(arg_tys
[1], "argument");
986 let (v_len
, _
) = arg_tys
[1].simd_size_and_type(bx
.tcx());
989 "mismatched lengths: mask length `{}` != other vector length `{}`",
993 match m_elem_ty
.kind() {
995 _
=> return_error
!("mask element type is `{}`, expected `i_`", m_elem_ty
),
997 // truncate the mask to a vector of i1s
998 let i1
= bx
.type_i1();
999 let i1xn
= bx
.type_vector(i1
, m_len
as u64);
1000 let m_i1s
= bx
.trunc(args
[0].immediate(), i1xn
);
1001 return Ok(bx
.select(m_i1s
, args
[1].immediate(), args
[2].immediate()));
1004 if name
== sym
::simd_bitmask
{
1005 // The `fn simd_bitmask(vector) -> unsigned integer` intrinsic takes a
1006 // vector mask and returns an unsigned integer containing the most
1007 // significant bit (MSB) of each lane.
1009 // If the vector has less than 8 lanes, an u8 is returned with zeroed
1011 let expected_int_bits
= in_len
.max(8);
1012 match ret_ty
.kind() {
1013 ty
::Uint(i
) if i
.bit_width() == Some(expected_int_bits
) => (),
1014 _
=> return_error
!("bitmask `{}`, expected `u{}`", ret_ty
, expected_int_bits
),
1017 // Integer vector <i{in_bitwidth} x in_len>:
1018 let (i_xn
, in_elem_bitwidth
) = match in_elem
.kind() {
1020 args
[0].immediate(),
1021 i
.bit_width().unwrap_or_else(|| bx
.data_layout().pointer_size
.bits()),
1024 args
[0].immediate(),
1025 i
.bit_width().unwrap_or_else(|| bx
.data_layout().pointer_size
.bits()),
1028 "vector argument `{}`'s element type `{}`, expected integer element type",
1034 // Shift the MSB to the right by "in_elem_bitwidth - 1" into the first bit position.
1037 bx
.cx
.const_int(bx
.type_ix(in_elem_bitwidth
), (in_elem_bitwidth
- 1) as _
);
1040 let i_xn_msb
= bx
.lshr(i_xn
, bx
.const_vector(shift_indices
.as_slice()));
1041 // Truncate vector to an <i1 x N>
1042 let i1xn
= bx
.trunc(i_xn_msb
, bx
.type_vector(bx
.type_i1(), in_len
));
1043 // Bitcast <i1 x N> to iN:
1044 let i_
= bx
.bitcast(i1xn
, bx
.type_ix(in_len
));
1045 // Zero-extend iN to the bitmask type:
1046 return Ok(bx
.zext(i_
, bx
.type_ix(expected_int_bits
)));
1049 fn simd_simple_float_intrinsic(
1051 in_elem
: &::rustc_middle
::ty
::TyS
<'_
>,
1052 in_ty
: &::rustc_middle
::ty
::TyS
<'_
>,
1054 bx
: &mut Builder
<'a
, 'll
, 'tcx
>,
1056 args
: &[OperandRef
<'tcx
, &'ll Value
>],
1057 ) -> Result
<&'ll Value
, ()> {
1058 macro_rules
! emit_error
{
1062 ($msg
: tt
, $
($fmt
: tt
)*) => {
1063 span_invalid_monomorphization_error(
1065 &format
!(concat
!("invalid monomorphization of `{}` intrinsic: ", $msg
),
1069 macro_rules
! return_error
{
1072 emit_error
!($
($fmt
)*);
1078 let (elem_ty_str
, elem_ty
) = if let ty
::Float(f
) = in_elem
.kind() {
1079 let elem_ty
= bx
.cx
.type_float_from_ty(*f
);
1080 match f
.bit_width() {
1081 32 => ("f32", elem_ty
),
1082 64 => ("f64", elem_ty
),
1085 "unsupported element type `{}` of floating-point vector `{}`",
1092 return_error
!("`{}` is not a floating-point type", in_ty
);
1095 let vec_ty
= bx
.type_vector(elem_ty
, in_len
);
1097 let (intr_name
, fn_ty
) = match name
{
1098 sym
::simd_ceil
=> ("ceil", bx
.type_func(&[vec_ty
], vec_ty
)),
1099 sym
::simd_fabs
=> ("fabs", bx
.type_func(&[vec_ty
], vec_ty
)),
1100 sym
::simd_fcos
=> ("cos", bx
.type_func(&[vec_ty
], vec_ty
)),
1101 sym
::simd_fexp2
=> ("exp2", bx
.type_func(&[vec_ty
], vec_ty
)),
1102 sym
::simd_fexp
=> ("exp", bx
.type_func(&[vec_ty
], vec_ty
)),
1103 sym
::simd_flog10
=> ("log10", bx
.type_func(&[vec_ty
], vec_ty
)),
1104 sym
::simd_flog2
=> ("log2", bx
.type_func(&[vec_ty
], vec_ty
)),
1105 sym
::simd_flog
=> ("log", bx
.type_func(&[vec_ty
], vec_ty
)),
1106 sym
::simd_floor
=> ("floor", bx
.type_func(&[vec_ty
], vec_ty
)),
1107 sym
::simd_fma
=> ("fma", bx
.type_func(&[vec_ty
, vec_ty
, vec_ty
], vec_ty
)),
1108 sym
::simd_fpowi
=> ("powi", bx
.type_func(&[vec_ty
, bx
.type_i32()], vec_ty
)),
1109 sym
::simd_fpow
=> ("pow", bx
.type_func(&[vec_ty
, vec_ty
], vec_ty
)),
1110 sym
::simd_fsin
=> ("sin", bx
.type_func(&[vec_ty
], vec_ty
)),
1111 sym
::simd_fsqrt
=> ("sqrt", bx
.type_func(&[vec_ty
], vec_ty
)),
1112 sym
::simd_round
=> ("round", bx
.type_func(&[vec_ty
], vec_ty
)),
1113 sym
::simd_trunc
=> ("trunc", bx
.type_func(&[vec_ty
], vec_ty
)),
1114 _
=> return_error
!("unrecognized intrinsic `{}`", name
),
1116 let llvm_name
= &format
!("llvm.{0}.v{1}{2}", intr_name
, in_len
, elem_ty_str
);
1117 let f
= bx
.declare_cfn(&llvm_name
, llvm
::UnnamedAddr
::No
, fn_ty
);
1118 let c
= bx
.call(f
, &args
.iter().map(|arg
| arg
.immediate()).collect
::<Vec
<_
>>(), None
);
1141 return simd_simple_float_intrinsic(name
, in_elem
, in_ty
, in_len
, bx
, span
, args
);
1145 // https://github.com/llvm-mirror/llvm/blob/master/include/llvm/IR/Function.h#L182
1146 // https://github.com/llvm-mirror/llvm/blob/master/include/llvm/IR/Intrinsics.h#L81
1147 fn llvm_vector_str(elem_ty
: Ty
<'_
>, vec_len
: u64, no_pointers
: usize) -> String
{
1148 let p0s
: String
= "p0".repeat(no_pointers
);
1149 match *elem_ty
.kind() {
1150 ty
::Int(v
) => format
!("v{}{}i{}", vec_len
, p0s
, v
.bit_width().unwrap()),
1151 ty
::Uint(v
) => format
!("v{}{}i{}", vec_len
, p0s
, v
.bit_width().unwrap()),
1152 ty
::Float(v
) => format
!("v{}{}f{}", vec_len
, p0s
, v
.bit_width()),
1153 _
=> unreachable
!(),
1158 cx
: &CodegenCx
<'ll
, '_
>,
1161 mut no_pointers
: usize,
1163 // FIXME: use cx.layout_of(ty).llvm_type() ?
1164 let mut elem_ty
= match *elem_ty
.kind() {
1165 ty
::Int(v
) => cx
.type_int_from_ty(v
),
1166 ty
::Uint(v
) => cx
.type_uint_from_ty(v
),
1167 ty
::Float(v
) => cx
.type_float_from_ty(v
),
1168 _
=> unreachable
!(),
1170 while no_pointers
> 0 {
1171 elem_ty
= cx
.type_ptr_to(elem_ty
);
1174 cx
.type_vector(elem_ty
, vec_len
)
1177 if name
== sym
::simd_gather
{
1178 // simd_gather(values: <N x T>, pointers: <N x *_ T>,
1179 // mask: <N x i{M}>) -> <N x T>
1180 // * N: number of elements in the input vectors
1181 // * T: type of the element to load
1182 // * M: any integer width is supported, will be truncated to i1
1184 // All types must be simd vector types
1185 require_simd
!(in_ty
, "first");
1186 require_simd
!(arg_tys
[1], "second");
1187 require_simd
!(arg_tys
[2], "third");
1188 require_simd
!(ret_ty
, "return");
1190 // Of the same length:
1191 let (out_len
, _
) = arg_tys
[1].simd_size_and_type(bx
.tcx());
1192 let (out_len2
, _
) = arg_tys
[2].simd_size_and_type(bx
.tcx());
1195 "expected {} argument with length {} (same as input type `{}`), \
1196 found `{}` with length {}",
1205 "expected {} argument with length {} (same as input type `{}`), \
1206 found `{}` with length {}",
1214 // The return type must match the first argument type
1215 require
!(ret_ty
== in_ty
, "expected return type `{}`, found `{}`", in_ty
, ret_ty
);
1217 // This counts how many pointers
1218 fn ptr_count(t
: Ty
<'_
>) -> usize {
1220 ty
::RawPtr(p
) => 1 + ptr_count(p
.ty
),
1226 fn non_ptr(t
: Ty
<'_
>) -> Ty
<'_
> {
1228 ty
::RawPtr(p
) => non_ptr(p
.ty
),
1233 // The second argument must be a simd vector with an element type that's a pointer
1234 // to the element type of the first argument
1235 let (_
, element_ty0
) = arg_tys
[0].simd_size_and_type(bx
.tcx());
1236 let (_
, element_ty1
) = arg_tys
[1].simd_size_and_type(bx
.tcx());
1237 let (pointer_count
, underlying_ty
) = match element_ty1
.kind() {
1238 ty
::RawPtr(p
) if p
.ty
== in_elem
=> (ptr_count(element_ty1
), non_ptr(element_ty1
)),
1242 "expected element type `{}` of second argument `{}` \
1243 to be a pointer to the element type `{}` of the first \
1244 argument `{}`, found `{}` != `*_ {}`",
1255 assert
!(pointer_count
> 0);
1256 assert_eq
!(pointer_count
- 1, ptr_count(element_ty0
));
1257 assert_eq
!(underlying_ty
, non_ptr(element_ty0
));
1259 // The element type of the third argument must be a signed integer type of any width:
1260 let (_
, element_ty2
) = arg_tys
[2].simd_size_and_type(bx
.tcx());
1261 match element_ty2
.kind() {
1266 "expected element type `{}` of third argument `{}` \
1267 to be a signed integer type",
1274 // Alignment of T, must be a constant integer value:
1275 let alignment_ty
= bx
.type_i32();
1276 let alignment
= bx
.const_i32(bx
.align_of(in_elem
).bytes() as i32);
1278 // Truncate the mask vector to a vector of i1s:
1279 let (mask
, mask_ty
) = {
1280 let i1
= bx
.type_i1();
1281 let i1xn
= bx
.type_vector(i1
, in_len
);
1282 (bx
.trunc(args
[2].immediate(), i1xn
), i1xn
)
1285 // Type of the vector of pointers:
1286 let llvm_pointer_vec_ty
= llvm_vector_ty(bx
, underlying_ty
, in_len
, pointer_count
);
1287 let llvm_pointer_vec_str
= llvm_vector_str(underlying_ty
, in_len
, pointer_count
);
1289 // Type of the vector of elements:
1290 let llvm_elem_vec_ty
= llvm_vector_ty(bx
, underlying_ty
, in_len
, pointer_count
- 1);
1291 let llvm_elem_vec_str
= llvm_vector_str(underlying_ty
, in_len
, pointer_count
- 1);
1293 let llvm_intrinsic
=
1294 format
!("llvm.masked.gather.{}.{}", llvm_elem_vec_str
, llvm_pointer_vec_str
);
1295 let f
= bx
.declare_cfn(
1297 llvm
::UnnamedAddr
::No
,
1299 &[llvm_pointer_vec_ty
, alignment_ty
, mask_ty
, llvm_elem_vec_ty
],
1303 let v
= bx
.call(f
, &[args
[1].immediate(), alignment
, mask
, args
[0].immediate()], None
);
1307 if name
== sym
::simd_scatter
{
1308 // simd_scatter(values: <N x T>, pointers: <N x *mut T>,
1309 // mask: <N x i{M}>) -> ()
1310 // * N: number of elements in the input vectors
1311 // * T: type of the element to load
1312 // * M: any integer width is supported, will be truncated to i1
1314 // All types must be simd vector types
1315 require_simd
!(in_ty
, "first");
1316 require_simd
!(arg_tys
[1], "second");
1317 require_simd
!(arg_tys
[2], "third");
1319 // Of the same length:
1320 let (element_len1
, _
) = arg_tys
[1].simd_size_and_type(bx
.tcx());
1321 let (element_len2
, _
) = arg_tys
[2].simd_size_and_type(bx
.tcx());
1323 in_len
== element_len1
,
1324 "expected {} argument with length {} (same as input type `{}`), \
1325 found `{}` with length {}",
1333 in_len
== element_len2
,
1334 "expected {} argument with length {} (same as input type `{}`), \
1335 found `{}` with length {}",
1343 // This counts how many pointers
1344 fn ptr_count(t
: Ty
<'_
>) -> usize {
1346 ty
::RawPtr(p
) => 1 + ptr_count(p
.ty
),
1352 fn non_ptr(t
: Ty
<'_
>) -> Ty
<'_
> {
1354 ty
::RawPtr(p
) => non_ptr(p
.ty
),
1359 // The second argument must be a simd vector with an element type that's a pointer
1360 // to the element type of the first argument
1361 let (_
, element_ty0
) = arg_tys
[0].simd_size_and_type(bx
.tcx());
1362 let (_
, element_ty1
) = arg_tys
[1].simd_size_and_type(bx
.tcx());
1363 let (_
, element_ty2
) = arg_tys
[2].simd_size_and_type(bx
.tcx());
1364 let (pointer_count
, underlying_ty
) = match element_ty1
.kind() {
1365 ty
::RawPtr(p
) if p
.ty
== in_elem
&& p
.mutbl
== hir
::Mutability
::Mut
=> {
1366 (ptr_count(element_ty1
), non_ptr(element_ty1
))
1371 "expected element type `{}` of second argument `{}` \
1372 to be a pointer to the element type `{}` of the first \
1373 argument `{}`, found `{}` != `*mut {}`",
1384 assert
!(pointer_count
> 0);
1385 assert_eq
!(pointer_count
- 1, ptr_count(element_ty0
));
1386 assert_eq
!(underlying_ty
, non_ptr(element_ty0
));
1388 // The element type of the third argument must be a signed integer type of any width:
1389 match element_ty2
.kind() {
1394 "expected element type `{}` of third argument `{}` \
1395 be a signed integer type",
1402 // Alignment of T, must be a constant integer value:
1403 let alignment_ty
= bx
.type_i32();
1404 let alignment
= bx
.const_i32(bx
.align_of(in_elem
).bytes() as i32);
1406 // Truncate the mask vector to a vector of i1s:
1407 let (mask
, mask_ty
) = {
1408 let i1
= bx
.type_i1();
1409 let i1xn
= bx
.type_vector(i1
, in_len
);
1410 (bx
.trunc(args
[2].immediate(), i1xn
), i1xn
)
1413 let ret_t
= bx
.type_void();
1415 // Type of the vector of pointers:
1416 let llvm_pointer_vec_ty
= llvm_vector_ty(bx
, underlying_ty
, in_len
, pointer_count
);
1417 let llvm_pointer_vec_str
= llvm_vector_str(underlying_ty
, in_len
, pointer_count
);
1419 // Type of the vector of elements:
1420 let llvm_elem_vec_ty
= llvm_vector_ty(bx
, underlying_ty
, in_len
, pointer_count
- 1);
1421 let llvm_elem_vec_str
= llvm_vector_str(underlying_ty
, in_len
, pointer_count
- 1);
1423 let llvm_intrinsic
=
1424 format
!("llvm.masked.scatter.{}.{}", llvm_elem_vec_str
, llvm_pointer_vec_str
);
1425 let f
= bx
.declare_cfn(
1427 llvm
::UnnamedAddr
::No
,
1428 bx
.type_func(&[llvm_elem_vec_ty
, llvm_pointer_vec_ty
, alignment_ty
, mask_ty
], ret_t
),
1430 let v
= bx
.call(f
, &[args
[0].immediate(), args
[1].immediate(), alignment
, mask
], None
);
1434 macro_rules
! arith_red
{
1435 ($name
:ident
: $integer_reduce
:ident
, $float_reduce
:ident
, $ordered
:expr
, $op
:ident
,
1436 $identity
:expr
) => {
1437 if name
== sym
::$name
{
1440 "expected return type `{}` (element of input `{}`), found `{}`",
1445 return match in_elem
.kind() {
1446 ty
::Int(_
) | ty
::Uint(_
) => {
1447 let r
= bx
.$
integer_reduce(args
[0].immediate());
1449 // if overflow occurs, the result is the
1450 // mathematical result modulo 2^n:
1451 Ok(bx
.$
op(args
[1].immediate(), r
))
1453 Ok(bx
.$
integer_reduce(args
[0].immediate()))
1457 let acc
= if $ordered
{
1458 // ordered arithmetic reductions take an accumulator
1461 // unordered arithmetic reductions use the identity accumulator
1462 match f
.bit_width() {
1463 32 => bx
.const_real(bx
.type_f32(), $identity
),
1464 64 => bx
.const_real(bx
.type_f64(), $identity
),
1467 unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#,
1476 Ok(bx
.$
float_reduce(acc
, args
[0].immediate()))
1479 "unsupported {} from `{}` with element `{}` to `{}`",
1490 arith_red
!(simd_reduce_add_ordered
: vector_reduce_add
, vector_reduce_fadd
, true, add
, 0.0);
1491 arith_red
!(simd_reduce_mul_ordered
: vector_reduce_mul
, vector_reduce_fmul
, true, mul
, 1.0);
1493 simd_reduce_add_unordered
: vector_reduce_add
,
1494 vector_reduce_fadd_fast
,
1500 simd_reduce_mul_unordered
: vector_reduce_mul
,
1501 vector_reduce_fmul_fast
,
1507 macro_rules
! minmax_red
{
1508 ($name
:ident
: $int_red
:ident
, $float_red
:ident
) => {
1509 if name
== sym
::$name
{
1512 "expected return type `{}` (element of input `{}`), found `{}`",
1517 return match in_elem
.kind() {
1518 ty
::Int(_i
) => Ok(bx
.$
int_red(args
[0].immediate(), true)),
1519 ty
::Uint(_u
) => Ok(bx
.$
int_red(args
[0].immediate(), false)),
1520 ty
::Float(_f
) => Ok(bx
.$
float_red(args
[0].immediate())),
1522 "unsupported {} from `{}` with element `{}` to `{}`",
1533 minmax_red
!(simd_reduce_min
: vector_reduce_min
, vector_reduce_fmin
);
1534 minmax_red
!(simd_reduce_max
: vector_reduce_max
, vector_reduce_fmax
);
1536 minmax_red
!(simd_reduce_min_nanless
: vector_reduce_min
, vector_reduce_fmin_fast
);
1537 minmax_red
!(simd_reduce_max_nanless
: vector_reduce_max
, vector_reduce_fmax_fast
);
1539 macro_rules
! bitwise_red
{
1540 ($name
:ident
: $red
:ident
, $boolean
:expr
) => {
1541 if name
== sym
::$name
{
1542 let input
= if !$boolean
{
1545 "expected return type `{}` (element of input `{}`), found `{}`",
1552 match in_elem
.kind() {
1553 ty
::Int(_
) | ty
::Uint(_
) => {}
1555 "unsupported {} from `{}` with element `{}` to `{}`",
1563 // boolean reductions operate on vectors of i1s:
1564 let i1
= bx
.type_i1();
1565 let i1xn
= bx
.type_vector(i1
, in_len
as u64);
1566 bx
.trunc(args
[0].immediate(), i1xn
)
1568 return match in_elem
.kind() {
1569 ty
::Int(_
) | ty
::Uint(_
) => {
1570 let r
= bx
.$
red(input
);
1571 Ok(if !$boolean { r }
else { bx.zext(r, bx.type_bool()) }
)
1574 "unsupported {} from `{}` with element `{}` to `{}`",
1585 bitwise_red
!(simd_reduce_and
: vector_reduce_and
, false);
1586 bitwise_red
!(simd_reduce_or
: vector_reduce_or
, false);
1587 bitwise_red
!(simd_reduce_xor
: vector_reduce_xor
, false);
1588 bitwise_red
!(simd_reduce_all
: vector_reduce_and
, true);
1589 bitwise_red
!(simd_reduce_any
: vector_reduce_or
, true);
1591 if name
== sym
::simd_cast
{
1592 require_simd
!(ret_ty
, "return");
1593 let (out_len
, out_elem
) = ret_ty
.simd_size_and_type(bx
.tcx());
1596 "expected return type with length {} (same as input type `{}`), \
1597 found `{}` with length {}",
1603 // casting cares about nominal type, not just structural type
1604 if in_elem
== out_elem
{
1605 return Ok(args
[0].immediate());
1610 Int(/* is signed? */ bool
),
1614 let (in_style
, in_width
) = match in_elem
.kind() {
1615 // vectors of pointer-sized integers should've been
1616 // disallowed before here, so this unwrap is safe.
1617 ty
::Int(i
) => (Style
::Int(true), i
.bit_width().unwrap()),
1618 ty
::Uint(u
) => (Style
::Int(false), u
.bit_width().unwrap()),
1619 ty
::Float(f
) => (Style
::Float
, f
.bit_width()),
1620 _
=> (Style
::Unsupported
, 0),
1622 let (out_style
, out_width
) = match out_elem
.kind() {
1623 ty
::Int(i
) => (Style
::Int(true), i
.bit_width().unwrap()),
1624 ty
::Uint(u
) => (Style
::Int(false), u
.bit_width().unwrap()),
1625 ty
::Float(f
) => (Style
::Float
, f
.bit_width()),
1626 _
=> (Style
::Unsupported
, 0),
1629 match (in_style
, out_style
) {
1630 (Style
::Int(in_is_signed
), Style
::Int(_
)) => {
1631 return Ok(match in_width
.cmp(&out_width
) {
1632 Ordering
::Greater
=> bx
.trunc(args
[0].immediate(), llret_ty
),
1633 Ordering
::Equal
=> args
[0].immediate(),
1636 bx
.sext(args
[0].immediate(), llret_ty
)
1638 bx
.zext(args
[0].immediate(), llret_ty
)
1643 (Style
::Int(in_is_signed
), Style
::Float
) => {
1644 return Ok(if in_is_signed
{
1645 bx
.sitofp(args
[0].immediate(), llret_ty
)
1647 bx
.uitofp(args
[0].immediate(), llret_ty
)
1650 (Style
::Float
, Style
::Int(out_is_signed
)) => {
1651 return Ok(if out_is_signed
{
1652 bx
.fptosi(args
[0].immediate(), llret_ty
)
1654 bx
.fptoui(args
[0].immediate(), llret_ty
)
1657 (Style
::Float
, Style
::Float
) => {
1658 return Ok(match in_width
.cmp(&out_width
) {
1659 Ordering
::Greater
=> bx
.fptrunc(args
[0].immediate(), llret_ty
),
1660 Ordering
::Equal
=> args
[0].immediate(),
1661 Ordering
::Less
=> bx
.fpext(args
[0].immediate(), llret_ty
),
1664 _
=> { /* Unsupported. Fallthrough. */ }
1668 "unsupported cast from `{}` with element `{}` to `{}` with element `{}`",
1675 macro_rules
! arith_binary
{
1676 ($
($name
: ident
: $
($
($p
: ident
),* => $call
: ident
),*;)*) => {
1677 $
(if name
== sym
::$name
{
1678 match in_elem
.kind() {
1679 $
($
(ty
::$
p(_
))|* => {
1680 return Ok(bx
.$
call(args
[0].immediate(), args
[1].immediate()))
1685 "unsupported operation on `{}` with element `{}`",
1692 simd_add
: Uint
, Int
=> add
, Float
=> fadd
;
1693 simd_sub
: Uint
, Int
=> sub
, Float
=> fsub
;
1694 simd_mul
: Uint
, Int
=> mul
, Float
=> fmul
;
1695 simd_div
: Uint
=> udiv
, Int
=> sdiv
, Float
=> fdiv
;
1696 simd_rem
: Uint
=> urem
, Int
=> srem
, Float
=> frem
;
1697 simd_shl
: Uint
, Int
=> shl
;
1698 simd_shr
: Uint
=> lshr
, Int
=> ashr
;
1699 simd_and
: Uint
, Int
=> and
;
1700 simd_or
: Uint
, Int
=> or
;
1701 simd_xor
: Uint
, Int
=> xor
;
1702 simd_fmax
: Float
=> maxnum
;
1703 simd_fmin
: Float
=> minnum
;
1706 macro_rules
! arith_unary
{
1707 ($
($name
: ident
: $
($
($p
: ident
),* => $call
: ident
),*;)*) => {
1708 $
(if name
== sym
::$name
{
1709 match in_elem
.kind() {
1710 $
($
(ty
::$
p(_
))|* => {
1711 return Ok(bx
.$
call(args
[0].immediate()))
1716 "unsupported operation on `{}` with element `{}`",
1723 simd_neg
: Int
=> neg
, Float
=> fneg
;
1726 if name
== sym
::simd_saturating_add
|| name
== sym
::simd_saturating_sub
{
1727 let lhs
= args
[0].immediate();
1728 let rhs
= args
[1].immediate();
1729 let is_add
= name
== sym
::simd_saturating_add
;
1730 let ptr_bits
= bx
.tcx().data_layout
.pointer_size
.bits() as _
;
1731 let (signed
, elem_width
, elem_ty
) = match *in_elem
.kind() {
1732 ty
::Int(i
) => (true, i
.bit_width().unwrap_or(ptr_bits
), bx
.cx
.type_int_from_ty(i
)),
1733 ty
::Uint(i
) => (false, i
.bit_width().unwrap_or(ptr_bits
), bx
.cx
.type_uint_from_ty(i
)),
1736 "expected element type `{}` of vector type `{}` \
1737 to be a signed or unsigned integer type",
1738 arg_tys
[0].simd_size_and_type(bx
.tcx()).1,
1743 let llvm_intrinsic
= &format
!(
1744 "llvm.{}{}.sat.v{}i{}",
1745 if signed { 's' }
else { 'u' }
,
1746 if is_add { "add" }
else { "sub" }
,
1750 let vec_ty
= bx
.cx
.type_vector(elem_ty
, in_len
as u64);
1752 let f
= bx
.declare_cfn(
1754 llvm
::UnnamedAddr
::No
,
1755 bx
.type_func(&[vec_ty
, vec_ty
], vec_ty
),
1757 let v
= bx
.call(f
, &[lhs
, rhs
], None
);
1761 span_bug
!(span
, "unknown SIMD intrinsic");
1764 // Returns the width of an int Ty, and if it's signed or not
1765 // Returns None if the type is not an integer
1766 // FIXME: there’s multiple of this functions, investigate using some of the already existing
1768 fn int_type_width_signed(ty
: Ty
<'_
>, cx
: &CodegenCx
<'_
, '_
>) -> Option
<(u64, bool
)> {
1771 Some((t
.bit_width().unwrap_or(u64::from(cx
.tcx
.sess
.target
.pointer_width
)), true))
1774 Some((t
.bit_width().unwrap_or(u64::from(cx
.tcx
.sess
.target
.pointer_width
)), false))