]> git.proxmox.com Git - rustc.git/blame - compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
Merge tag 'debian/1.52.1+dfsg1-1_exp2' into proxmox/buster
[rustc.git] / compiler / rustc_codegen_cranelift / src / intrinsics / simd.rs
CommitLineData
29967ef6
XL
1//! Codegen `extern "platform-intrinsic"` intrinsics.
2
3use super::*;
4use crate::prelude::*;
5
6pub(super) fn codegen_simd_intrinsic_call<'tcx>(
6a06907d 7 fx: &mut FunctionCx<'_, '_, 'tcx>,
29967ef6
XL
8 instance: Instance<'tcx>,
9 args: &[mir::Operand<'tcx>],
10 ret: CPlace<'tcx>,
11 span: Span,
12) {
13 let def_id = instance.def_id();
14 let substs = instance.substs;
15
16 let intrinsic = fx.tcx.item_name(def_id).as_str();
17 let intrinsic = &intrinsic[..];
18
19 intrinsic_match! {
20 fx, intrinsic, substs, args,
21 _ => {
22 fx.tcx.sess.span_fatal(span, &format!("Unknown SIMD intrinsic {}", intrinsic));
23 };
24
25 simd_cast, (c a) {
26 validate_simd_type!(fx, intrinsic, span, a.layout().ty);
27 simd_for_each_lane(fx, a, ret, |fx, lane_layout, ret_lane_layout, lane| {
28 let ret_lane_ty = fx.clif_type(ret_lane_layout.ty).unwrap();
29
30 let from_signed = type_sign(lane_layout.ty);
31 let to_signed = type_sign(ret_lane_layout.ty);
32
33 let ret_lane = clif_int_or_float_cast(fx, lane, from_signed, ret_lane_ty, to_signed);
34 CValue::by_val(ret_lane, ret_lane_layout)
35 });
36 };
37
29967ef6
XL
38 simd_eq, (c x, c y) {
39 validate_simd_type!(fx, intrinsic, span, x.layout().ty);
fc512014 40 simd_cmp!(fx, Equal|Equal(x, y) -> ret);
29967ef6
XL
41 };
42 simd_ne, (c x, c y) {
43 validate_simd_type!(fx, intrinsic, span, x.layout().ty);
fc512014 44 simd_cmp!(fx, NotEqual|NotEqual(x, y) -> ret);
29967ef6
XL
45 };
46 simd_lt, (c x, c y) {
47 validate_simd_type!(fx, intrinsic, span, x.layout().ty);
fc512014 48 simd_cmp!(fx, UnsignedLessThan|SignedLessThan|LessThan(x, y) -> ret);
29967ef6
XL
49 };
50 simd_le, (c x, c y) {
51 validate_simd_type!(fx, intrinsic, span, x.layout().ty);
fc512014 52 simd_cmp!(fx, UnsignedLessThanOrEqual|SignedLessThanOrEqual|LessThanOrEqual(x, y) -> ret);
29967ef6
XL
53 };
54 simd_gt, (c x, c y) {
55 validate_simd_type!(fx, intrinsic, span, x.layout().ty);
fc512014 56 simd_cmp!(fx, UnsignedGreaterThan|SignedGreaterThan|GreaterThan(x, y) -> ret);
29967ef6
XL
57 };
58 simd_ge, (c x, c y) {
59 validate_simd_type!(fx, intrinsic, span, x.layout().ty);
fc512014
XL
60 simd_cmp!(
61 fx,
62 UnsignedGreaterThanOrEqual|SignedGreaterThanOrEqual|GreaterThanOrEqual
63 (x, y) -> ret
64 );
29967ef6
XL
65 };
66
67 // simd_shuffle32<T, U>(x: T, y: T, idx: [u32; 32]) -> U
68 _ if intrinsic.starts_with("simd_shuffle"), (c x, c y, o idx) {
69 validate_simd_type!(fx, intrinsic, span, x.layout().ty);
70
71 let n: u16 = intrinsic["simd_shuffle".len()..].parse().unwrap();
72
73 assert_eq!(x.layout(), y.layout());
74 let layout = x.layout();
75
5869c6ff
XL
76 let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
77 let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
29967ef6 78
5869c6ff
XL
79 assert_eq!(lane_ty, ret_lane_ty);
80 assert_eq!(u64::from(n), ret_lane_count);
29967ef6
XL
81
82 let total_len = lane_count * 2;
83
84 let indexes = {
85 use rustc_middle::mir::interpret::*;
86 let idx_const = crate::constant::mir_operand_get_const_val(fx, idx).expect("simd_shuffle* idx not const");
87
6a06907d
XL
88 let idx_bytes = match idx_const {
89 ConstValue::ByRef { alloc, offset } => {
29967ef6
XL
90 let ptr = Pointer::new(AllocId(0 /* dummy */), offset);
91 let size = Size::from_bytes(4 * u64::from(ret_lane_count) /* size_of([u32; ret_lane_count]) */);
92 alloc.get_bytes(fx, ptr, size).unwrap()
93 }
94 _ => unreachable!("{:?}", idx_const),
95 };
96
97 (0..ret_lane_count).map(|i| {
98 let i = usize::try_from(i).unwrap();
99 let idx = rustc_middle::mir::interpret::read_target_uint(
100 fx.tcx.data_layout.endian,
101 &idx_bytes[4*i.. 4*i + 4],
102 ).expect("read_target_uint");
103 u16::try_from(idx).expect("try_from u32")
104 }).collect::<Vec<u16>>()
105 };
106
107 for &idx in &indexes {
5869c6ff 108 assert!(u64::from(idx) < total_len, "idx {} out of range 0..{}", idx, total_len);
29967ef6
XL
109 }
110
111 for (out_idx, in_idx) in indexes.into_iter().enumerate() {
5869c6ff 112 let in_lane = if u64::from(in_idx) < lane_count {
fc512014 113 x.value_field(fx, mir::Field::new(in_idx.into()))
29967ef6 114 } else {
5869c6ff 115 y.value_field(fx, mir::Field::new(usize::from(in_idx) - usize::try_from(lane_count).unwrap()))
29967ef6
XL
116 };
117 let out_lane = ret.place_field(fx, mir::Field::new(out_idx));
118 out_lane.write_cvalue(fx, in_lane);
119 }
120 };
121
122 simd_insert, (c base, o idx, c val) {
123 // FIXME validate
124 let idx_const = if let Some(idx_const) = crate::constant::mir_operand_get_const_val(fx, idx) {
125 idx_const
126 } else {
127 fx.tcx.sess.span_fatal(
128 span,
129 "Index argument for `simd_insert` is not a constant",
130 );
131 };
132
6a06907d 133 let idx = idx_const.try_to_bits(Size::from_bytes(4 /* u32*/)).unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const));
5869c6ff 134 let (lane_count, _lane_ty) = base.layout().ty.simd_size_and_type(fx.tcx);
29967ef6
XL
135 if idx >= lane_count.into() {
136 fx.tcx.sess.span_fatal(fx.mir.span, &format!("[simd_insert] idx {} >= lane_count {}", idx, lane_count));
137 }
138
139 ret.write_cvalue(fx, base);
140 let ret_lane = ret.place_field(fx, mir::Field::new(idx.try_into().unwrap()));
141 ret_lane.write_cvalue(fx, val);
142 };
143
144 simd_extract, (c v, o idx) {
145 validate_simd_type!(fx, intrinsic, span, v.layout().ty);
146 let idx_const = if let Some(idx_const) = crate::constant::mir_operand_get_const_val(fx, idx) {
147 idx_const
148 } else {
fc512014 149 fx.tcx.sess.span_warn(
29967ef6
XL
150 span,
151 "Index argument for `simd_extract` is not a constant",
152 );
fc512014
XL
153 let res = crate::trap::trap_unimplemented_ret_value(
154 fx,
155 ret.layout(),
156 "Index argument for `simd_extract` is not a constant",
157 );
158 ret.write_cvalue(fx, res);
159 return;
29967ef6
XL
160 };
161
6a06907d 162 let idx = idx_const.try_to_bits(Size::from_bytes(4 /* u32*/)).unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const));
5869c6ff 163 let (lane_count, _lane_ty) = v.layout().ty.simd_size_and_type(fx.tcx);
29967ef6
XL
164 if idx >= lane_count.into() {
165 fx.tcx.sess.span_fatal(fx.mir.span, &format!("[simd_extract] idx {} >= lane_count {}", idx, lane_count));
166 }
167
168 let ret_lane = v.value_field(fx, mir::Field::new(idx.try_into().unwrap()));
169 ret.write_cvalue(fx, ret_lane);
170 };
171
172 simd_add, (c x, c y) {
173 validate_simd_type!(fx, intrinsic, span, x.layout().ty);
174 simd_int_flt_binop!(fx, iadd|fadd(x, y) -> ret);
175 };
176 simd_sub, (c x, c y) {
177 validate_simd_type!(fx, intrinsic, span, x.layout().ty);
178 simd_int_flt_binop!(fx, isub|fsub(x, y) -> ret);
179 };
180 simd_mul, (c x, c y) {
181 validate_simd_type!(fx, intrinsic, span, x.layout().ty);
182 simd_int_flt_binop!(fx, imul|fmul(x, y) -> ret);
183 };
184 simd_div, (c x, c y) {
185 validate_simd_type!(fx, intrinsic, span, x.layout().ty);
186 simd_int_flt_binop!(fx, udiv|sdiv|fdiv(x, y) -> ret);
187 };
188 simd_shl, (c x, c y) {
189 validate_simd_type!(fx, intrinsic, span, x.layout().ty);
190 simd_int_binop!(fx, ishl(x, y) -> ret);
191 };
192 simd_shr, (c x, c y) {
193 validate_simd_type!(fx, intrinsic, span, x.layout().ty);
194 simd_int_binop!(fx, ushr|sshr(x, y) -> ret);
195 };
196 simd_and, (c x, c y) {
197 validate_simd_type!(fx, intrinsic, span, x.layout().ty);
198 simd_int_binop!(fx, band(x, y) -> ret);
199 };
200 simd_or, (c x, c y) {
201 validate_simd_type!(fx, intrinsic, span, x.layout().ty);
202 simd_int_binop!(fx, bor(x, y) -> ret);
203 };
204 simd_xor, (c x, c y) {
205 validate_simd_type!(fx, intrinsic, span, x.layout().ty);
206 simd_int_binop!(fx, bxor(x, y) -> ret);
207 };
208
209 simd_fma, (c a, c b, c c) {
210 validate_simd_type!(fx, intrinsic, span, a.layout().ty);
211 assert_eq!(a.layout(), b.layout());
212 assert_eq!(a.layout(), c.layout());
213 let layout = a.layout();
214
5869c6ff
XL
215 let (lane_count, _lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
216 let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx);
29967ef6 217 assert_eq!(lane_count, ret_lane_count);
5869c6ff 218 let ret_lane_layout = fx.layout_of(ret_lane_ty);
29967ef6
XL
219
220 for lane in 0..lane_count {
5869c6ff 221 let lane = mir::Field::new(lane.try_into().unwrap());
29967ef6
XL
222 let a_lane = a.value_field(fx, lane).load_scalar(fx);
223 let b_lane = b.value_field(fx, lane).load_scalar(fx);
224 let c_lane = c.value_field(fx, lane).load_scalar(fx);
225
226 let mul_lane = fx.bcx.ins().fmul(a_lane, b_lane);
227 let res_lane = CValue::by_val(fx.bcx.ins().fadd(mul_lane, c_lane), ret_lane_layout);
228
229 ret.place_field(fx, lane).write_cvalue(fx, res_lane);
230 }
231 };
232
233 simd_fmin, (c x, c y) {
234 validate_simd_type!(fx, intrinsic, span, x.layout().ty);
235 simd_flt_binop!(fx, fmin(x, y) -> ret);
236 };
237 simd_fmax, (c x, c y) {
238 validate_simd_type!(fx, intrinsic, span, x.layout().ty);
239 simd_flt_binop!(fx, fmax(x, y) -> ret);
240 };
241
fc512014
XL
242 simd_reduce_add_ordered | simd_reduce_add_unordered, (c v) {
243 validate_simd_type!(fx, intrinsic, span, v.layout().ty);
244 simd_reduce(fx, v, ret, |fx, lane_layout, a, b| {
245 if lane_layout.ty.is_floating_point() {
246 fx.bcx.ins().fadd(a, b)
247 } else {
248 fx.bcx.ins().iadd(a, b)
249 }
250 });
251 };
252
253 simd_reduce_mul_ordered | simd_reduce_mul_unordered, (c v) {
254 validate_simd_type!(fx, intrinsic, span, v.layout().ty);
255 simd_reduce(fx, v, ret, |fx, lane_layout, a, b| {
256 if lane_layout.ty.is_floating_point() {
257 fx.bcx.ins().fmul(a, b)
258 } else {
259 fx.bcx.ins().imul(a, b)
260 }
261 });
262 };
263
264 simd_reduce_all, (c v) {
265 validate_simd_type!(fx, intrinsic, span, v.layout().ty);
266 simd_reduce_bool(fx, v, ret, |fx, a, b| fx.bcx.ins().band(a, b));
267 };
268
269 simd_reduce_any, (c v) {
270 validate_simd_type!(fx, intrinsic, span, v.layout().ty);
271 simd_reduce_bool(fx, v, ret, |fx, a, b| fx.bcx.ins().bor(a, b));
272 };
273
29967ef6
XL
274 // simd_fabs
275 // simd_saturating_add
276 // simd_bitmask
277 // simd_select
29967ef6 278 // simd_rem
6a06907d 279 // simd_neg
29967ef6
XL
280 }
281}