]>
Commit | Line | Data |
---|---|---|
29967ef6 XL |
1 | //! Definition of [`CValue`] and [`CPlace`] |
2 | ||
3 | use crate::prelude::*; | |
4 | ||
29967ef6 XL |
5 | use cranelift_codegen::ir::immediates::Offset32; |
6 | ||
7 | fn codegen_field<'tcx>( | |
6a06907d | 8 | fx: &mut FunctionCx<'_, '_, 'tcx>, |
29967ef6 XL |
9 | base: Pointer, |
10 | extra: Option<Value>, | |
11 | layout: TyAndLayout<'tcx>, | |
12 | field: mir::Field, | |
13 | ) -> (Pointer, TyAndLayout<'tcx>) { | |
14 | let field_offset = layout.fields.offset(field.index()); | |
15 | let field_layout = layout.field(&*fx, field.index()); | |
16 | ||
6a06907d XL |
17 | let simple = |fx: &mut FunctionCx<'_, '_, '_>| { |
18 | (base.offset_i64(fx, i64::try_from(field_offset.bytes()).unwrap()), field_layout) | |
29967ef6 XL |
19 | }; |
20 | ||
21 | if let Some(extra) = extra { | |
22 | if !field_layout.is_unsized() { | |
23 | return simple(fx); | |
24 | } | |
25 | match field_layout.ty.kind() { | |
26 | ty::Slice(..) | ty::Str | ty::Foreign(..) => simple(fx), | |
5e7ed085 | 27 | ty::Adt(def, _) if def.repr().packed() => { |
29967ef6 XL |
28 | assert_eq!(layout.align.abi.bytes(), 1); |
29 | simple(fx) | |
30 | } | |
31 | _ => { | |
32 | // We have to align the offset for DST's | |
33 | let unaligned_offset = field_offset.bytes(); | |
34 | let (_, unsized_align) = | |
35 | crate::unsize::size_and_align_of_dst(fx, field_layout, extra); | |
36 | ||
94222f64 | 37 | let one = fx.bcx.ins().iconst(fx.pointer_type, 1); |
29967ef6 XL |
38 | let align_sub_1 = fx.bcx.ins().isub(unsized_align, one); |
39 | let and_lhs = fx.bcx.ins().iadd_imm(align_sub_1, unaligned_offset as i64); | |
94222f64 | 40 | let zero = fx.bcx.ins().iconst(fx.pointer_type, 0); |
29967ef6 XL |
41 | let and_rhs = fx.bcx.ins().isub(zero, unsized_align); |
42 | let offset = fx.bcx.ins().band(and_lhs, and_rhs); | |
43 | ||
44 | (base.offset_value(fx, offset), field_layout) | |
45 | } | |
46 | } | |
47 | } else { | |
48 | simple(fx) | |
49 | } | |
50 | } | |
51 | ||
c295e0f8 | 52 | fn scalar_pair_calculate_b_offset(tcx: TyCtxt<'_>, a_scalar: Scalar, b_scalar: Scalar) -> Offset32 { |
04454e1e | 53 | let b_offset = a_scalar.size(&tcx).align_to(b_scalar.align(&tcx).abi); |
29967ef6 XL |
54 | Offset32::new(b_offset.bytes().try_into().unwrap()) |
55 | } | |
56 | ||
57 | /// A read-only value | |
58 | #[derive(Debug, Copy, Clone)] | |
59 | pub(crate) struct CValue<'tcx>(CValueInner, TyAndLayout<'tcx>); | |
60 | ||
61 | #[derive(Debug, Copy, Clone)] | |
62 | enum CValueInner { | |
63 | ByRef(Pointer, Option<Value>), | |
64 | ByVal(Value), | |
65 | ByValPair(Value, Value), | |
66 | } | |
67 | ||
68 | impl<'tcx> CValue<'tcx> { | |
69 | pub(crate) fn by_ref(ptr: Pointer, layout: TyAndLayout<'tcx>) -> CValue<'tcx> { | |
70 | CValue(CValueInner::ByRef(ptr, None), layout) | |
71 | } | |
72 | ||
73 | pub(crate) fn by_ref_unsized( | |
74 | ptr: Pointer, | |
75 | meta: Value, | |
76 | layout: TyAndLayout<'tcx>, | |
77 | ) -> CValue<'tcx> { | |
78 | CValue(CValueInner::ByRef(ptr, Some(meta)), layout) | |
79 | } | |
80 | ||
81 | pub(crate) fn by_val(value: Value, layout: TyAndLayout<'tcx>) -> CValue<'tcx> { | |
82 | CValue(CValueInner::ByVal(value), layout) | |
83 | } | |
84 | ||
85 | pub(crate) fn by_val_pair( | |
86 | value: Value, | |
87 | extra: Value, | |
88 | layout: TyAndLayout<'tcx>, | |
89 | ) -> CValue<'tcx> { | |
90 | CValue(CValueInner::ByValPair(value, extra), layout) | |
91 | } | |
92 | ||
93 | pub(crate) fn layout(&self) -> TyAndLayout<'tcx> { | |
94 | self.1 | |
95 | } | |
96 | ||
97 | // FIXME remove | |
6a06907d | 98 | pub(crate) fn force_stack(self, fx: &mut FunctionCx<'_, '_, 'tcx>) -> (Pointer, Option<Value>) { |
29967ef6 XL |
99 | let layout = self.1; |
100 | match self.0 { | |
101 | CValueInner::ByRef(ptr, meta) => (ptr, meta), | |
102 | CValueInner::ByVal(_) | CValueInner::ByValPair(_, _) => { | |
103 | let cplace = CPlace::new_stack_slot(fx, layout); | |
104 | cplace.write_cvalue(fx, self); | |
105 | (cplace.to_ptr(), None) | |
106 | } | |
107 | } | |
108 | } | |
109 | ||
110 | pub(crate) fn try_to_ptr(self) -> Option<(Pointer, Option<Value>)> { | |
111 | match self.0 { | |
112 | CValueInner::ByRef(ptr, meta) => Some((ptr, meta)), | |
113 | CValueInner::ByVal(_) | CValueInner::ByValPair(_, _) => None, | |
114 | } | |
115 | } | |
116 | ||
117 | /// Load a value with layout.abi of scalar | |
6a06907d | 118 | pub(crate) fn load_scalar(self, fx: &mut FunctionCx<'_, '_, 'tcx>) -> Value { |
29967ef6 XL |
119 | let layout = self.1; |
120 | match self.0 { | |
121 | CValueInner::ByRef(ptr, None) => { | |
122 | let clif_ty = match layout.abi { | |
c295e0f8 XL |
123 | Abi::Scalar(scalar) => scalar_to_clif_type(fx.tcx, scalar), |
124 | Abi::Vector { element, count } => scalar_to_clif_type(fx.tcx, element) | |
125 | .by(u16::try_from(count).unwrap()) | |
126 | .unwrap(), | |
29967ef6 XL |
127 | _ => unreachable!("{:?}", layout.ty), |
128 | }; | |
129 | let mut flags = MemFlags::new(); | |
130 | flags.set_notrap(); | |
131 | ptr.load(fx, clif_ty, flags) | |
132 | } | |
133 | CValueInner::ByVal(value) => value, | |
134 | CValueInner::ByRef(_, Some(_)) => bug!("load_scalar for unsized value not allowed"), | |
135 | CValueInner::ByValPair(_, _) => bug!("Please use load_scalar_pair for ByValPair"), | |
136 | } | |
137 | } | |
138 | ||
139 | /// Load a value pair with layout.abi of scalar pair | |
6a06907d | 140 | pub(crate) fn load_scalar_pair(self, fx: &mut FunctionCx<'_, '_, 'tcx>) -> (Value, Value) { |
29967ef6 XL |
141 | let layout = self.1; |
142 | match self.0 { | |
143 | CValueInner::ByRef(ptr, None) => { | |
c295e0f8 | 144 | let (a_scalar, b_scalar) = match layout.abi { |
29967ef6 XL |
145 | Abi::ScalarPair(a, b) => (a, b), |
146 | _ => unreachable!("load_scalar_pair({:?})", self), | |
147 | }; | |
148 | let b_offset = scalar_pair_calculate_b_offset(fx.tcx, a_scalar, b_scalar); | |
c295e0f8 XL |
149 | let clif_ty1 = scalar_to_clif_type(fx.tcx, a_scalar); |
150 | let clif_ty2 = scalar_to_clif_type(fx.tcx, b_scalar); | |
29967ef6 XL |
151 | let mut flags = MemFlags::new(); |
152 | flags.set_notrap(); | |
153 | let val1 = ptr.load(fx, clif_ty1, flags); | |
154 | let val2 = ptr.offset(fx, b_offset).load(fx, clif_ty2, flags); | |
155 | (val1, val2) | |
156 | } | |
157 | CValueInner::ByRef(_, Some(_)) => { | |
158 | bug!("load_scalar_pair for unsized value not allowed") | |
159 | } | |
160 | CValueInner::ByVal(_) => bug!("Please use load_scalar for ByVal"), | |
161 | CValueInner::ByValPair(val1, val2) => (val1, val2), | |
162 | } | |
163 | } | |
164 | ||
165 | pub(crate) fn value_field( | |
166 | self, | |
6a06907d | 167 | fx: &mut FunctionCx<'_, '_, 'tcx>, |
29967ef6 XL |
168 | field: mir::Field, |
169 | ) -> CValue<'tcx> { | |
170 | let layout = self.1; | |
171 | match self.0 { | |
172 | CValueInner::ByVal(val) => match layout.abi { | |
173 | Abi::Vector { element: _, count } => { | |
174 | let count = u8::try_from(count).expect("SIMD type with more than 255 lanes???"); | |
175 | let field = u8::try_from(field.index()).unwrap(); | |
176 | assert!(field < count); | |
177 | let lane = fx.bcx.ins().extractlane(val, field); | |
178 | let field_layout = layout.field(&*fx, usize::from(field)); | |
179 | CValue::by_val(lane, field_layout) | |
180 | } | |
181 | _ => unreachable!("value_field for ByVal with abi {:?}", layout.abi), | |
182 | }, | |
183 | CValueInner::ByValPair(val1, val2) => match layout.abi { | |
184 | Abi::ScalarPair(_, _) => { | |
185 | let val = match field.as_u32() { | |
186 | 0 => val1, | |
187 | 1 => val2, | |
188 | _ => bug!("field should be 0 or 1"), | |
189 | }; | |
190 | let field_layout = layout.field(&*fx, usize::from(field)); | |
191 | CValue::by_val(val, field_layout) | |
192 | } | |
193 | _ => unreachable!("value_field for ByValPair with abi {:?}", layout.abi), | |
194 | }, | |
195 | CValueInner::ByRef(ptr, None) => { | |
196 | let (field_ptr, field_layout) = codegen_field(fx, ptr, None, layout, field); | |
197 | CValue::by_ref(field_ptr, field_layout) | |
198 | } | |
199 | CValueInner::ByRef(_, Some(_)) => todo!(), | |
200 | } | |
201 | } | |
202 | ||
94222f64 XL |
203 | /// Like [`CValue::value_field`] except handling ADTs containing a single array field in a way |
204 | /// such that you can access individual lanes. | |
205 | pub(crate) fn value_lane( | |
206 | self, | |
207 | fx: &mut FunctionCx<'_, '_, 'tcx>, | |
208 | lane_idx: u64, | |
209 | ) -> CValue<'tcx> { | |
210 | let layout = self.1; | |
211 | assert!(layout.ty.is_simd()); | |
212 | let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx); | |
213 | let lane_layout = fx.layout_of(lane_ty); | |
214 | assert!(lane_idx < lane_count); | |
215 | match self.0 { | |
216 | CValueInner::ByVal(val) => match layout.abi { | |
217 | Abi::Vector { element: _, count: _ } => { | |
218 | assert!(lane_count <= u8::MAX.into(), "SIMD type with more than 255 lanes???"); | |
219 | let lane_idx = u8::try_from(lane_idx).unwrap(); | |
220 | let lane = fx.bcx.ins().extractlane(val, lane_idx); | |
221 | CValue::by_val(lane, lane_layout) | |
222 | } | |
223 | _ => unreachable!("value_lane for ByVal with abi {:?}", layout.abi), | |
224 | }, | |
225 | CValueInner::ByValPair(_, _) => unreachable!(), | |
226 | CValueInner::ByRef(ptr, None) => { | |
227 | let field_offset = lane_layout.size * lane_idx; | |
228 | let field_ptr = ptr.offset_i64(fx, i64::try_from(field_offset.bytes()).unwrap()); | |
229 | CValue::by_ref(field_ptr, lane_layout) | |
230 | } | |
231 | CValueInner::ByRef(_, Some(_)) => unreachable!(), | |
232 | } | |
233 | } | |
234 | ||
6a06907d | 235 | pub(crate) fn unsize_value(self, fx: &mut FunctionCx<'_, '_, 'tcx>, dest: CPlace<'tcx>) { |
29967ef6 XL |
236 | crate::unsize::coerce_unsized_into(fx, self, dest); |
237 | } | |
238 | ||
239 | /// If `ty` is signed, `const_val` must already be sign extended. | |
240 | pub(crate) fn const_val( | |
6a06907d | 241 | fx: &mut FunctionCx<'_, '_, 'tcx>, |
29967ef6 XL |
242 | layout: TyAndLayout<'tcx>, |
243 | const_val: ty::ScalarInt, | |
244 | ) -> CValue<'tcx> { | |
6a06907d | 245 | assert_eq!(const_val.size(), layout.size, "{:#?}: {:?}", const_val, layout); |
29967ef6 XL |
246 | use cranelift_codegen::ir::immediates::{Ieee32, Ieee64}; |
247 | ||
248 | let clif_ty = fx.clif_type(layout.ty).unwrap(); | |
249 | ||
250 | if let ty::Bool = layout.ty.kind() { | |
251 | assert!( | |
252 | const_val == ty::ScalarInt::FALSE || const_val == ty::ScalarInt::TRUE, | |
253 | "Invalid bool 0x{:032X}", | |
254 | const_val | |
255 | ); | |
256 | } | |
257 | ||
258 | let val = match layout.ty.kind() { | |
259 | ty::Uint(UintTy::U128) | ty::Int(IntTy::I128) => { | |
260 | let const_val = const_val.to_bits(layout.size).unwrap(); | |
261 | let lsb = fx.bcx.ins().iconst(types::I64, const_val as u64 as i64); | |
6a06907d | 262 | let msb = fx.bcx.ins().iconst(types::I64, (const_val >> 64) as u64 as i64); |
29967ef6 XL |
263 | fx.bcx.ins().iconcat(lsb, msb) |
264 | } | |
6a06907d XL |
265 | ty::Bool | ty::Char | ty::Uint(_) | ty::Int(_) | ty::Ref(..) | ty::RawPtr(..) => { |
266 | fx.bcx.ins().iconst(clif_ty, const_val.to_bits(layout.size).unwrap() as i64) | |
29967ef6 XL |
267 | } |
268 | ty::Float(FloatTy::F32) => { | |
269 | fx.bcx.ins().f32const(Ieee32::with_bits(u32::try_from(const_val).unwrap())) | |
270 | } | |
271 | ty::Float(FloatTy::F64) => { | |
272 | fx.bcx.ins().f64const(Ieee64::with_bits(u64::try_from(const_val).unwrap())) | |
273 | } | |
274 | _ => panic!( | |
275 | "CValue::const_val for non bool/char/float/integer/pointer type {:?} is not allowed", | |
276 | layout.ty | |
277 | ), | |
278 | }; | |
279 | ||
280 | CValue::by_val(val, layout) | |
281 | } | |
282 | ||
283 | pub(crate) fn cast_pointer_to(self, layout: TyAndLayout<'tcx>) -> Self { | |
6a06907d XL |
284 | assert!(matches!(self.layout().ty.kind(), ty::Ref(..) | ty::RawPtr(..) | ty::FnPtr(..))); |
285 | assert!(matches!(layout.ty.kind(), ty::Ref(..) | ty::RawPtr(..) | ty::FnPtr(..))); | |
29967ef6 XL |
286 | assert_eq!(self.layout().abi, layout.abi); |
287 | CValue(self.0, layout) | |
288 | } | |
289 | } | |
290 | ||
291 | /// A place where you can write a value to or read a value from | |
292 | #[derive(Debug, Copy, Clone)] | |
293 | pub(crate) struct CPlace<'tcx> { | |
294 | inner: CPlaceInner, | |
295 | layout: TyAndLayout<'tcx>, | |
296 | } | |
297 | ||
298 | #[derive(Debug, Copy, Clone)] | |
299 | pub(crate) enum CPlaceInner { | |
300 | Var(Local, Variable), | |
301 | VarPair(Local, Variable, Variable), | |
302 | VarLane(Local, Variable, u8), | |
303 | Addr(Pointer, Option<Value>), | |
304 | } | |
305 | ||
306 | impl<'tcx> CPlace<'tcx> { | |
307 | pub(crate) fn layout(&self) -> TyAndLayout<'tcx> { | |
308 | self.layout | |
309 | } | |
310 | ||
311 | pub(crate) fn inner(&self) -> &CPlaceInner { | |
312 | &self.inner | |
313 | } | |
314 | ||
29967ef6 | 315 | pub(crate) fn new_stack_slot( |
6a06907d | 316 | fx: &mut FunctionCx<'_, '_, 'tcx>, |
29967ef6 XL |
317 | layout: TyAndLayout<'tcx>, |
318 | ) -> CPlace<'tcx> { | |
319 | assert!(!layout.is_unsized()); | |
320 | if layout.size.bytes() == 0 { | |
94222f64 XL |
321 | return CPlace { |
322 | inner: CPlaceInner::Addr(Pointer::dangling(layout.align.pref), None), | |
323 | layout, | |
324 | }; | |
29967ef6 XL |
325 | } |
326 | ||
064997fb FG |
327 | if layout.size.bytes() >= u64::from(u32::MAX - 16) { |
328 | fx.tcx | |
329 | .sess | |
330 | .fatal(&format!("values of type {} are too big to store on the stack", layout.ty)); | |
331 | } | |
332 | ||
29967ef6 XL |
333 | let stack_slot = fx.bcx.create_stack_slot(StackSlotData { |
334 | kind: StackSlotKind::ExplicitSlot, | |
5869c6ff XL |
335 | // FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to |
336 | // specify stack slot alignment. | |
337 | size: (u32::try_from(layout.size.bytes()).unwrap() + 15) / 16 * 16, | |
29967ef6 | 338 | }); |
6a06907d | 339 | CPlace { inner: CPlaceInner::Addr(Pointer::stack_slot(stack_slot), None), layout } |
29967ef6 XL |
340 | } |
341 | ||
342 | pub(crate) fn new_var( | |
6a06907d | 343 | fx: &mut FunctionCx<'_, '_, 'tcx>, |
29967ef6 XL |
344 | local: Local, |
345 | layout: TyAndLayout<'tcx>, | |
346 | ) -> CPlace<'tcx> { | |
347 | let var = Variable::with_u32(fx.next_ssa_var); | |
348 | fx.next_ssa_var += 1; | |
349 | fx.bcx.declare_var(var, fx.clif_type(layout.ty).unwrap()); | |
6a06907d | 350 | CPlace { inner: CPlaceInner::Var(local, var), layout } |
29967ef6 XL |
351 | } |
352 | ||
353 | pub(crate) fn new_var_pair( | |
6a06907d | 354 | fx: &mut FunctionCx<'_, '_, 'tcx>, |
29967ef6 XL |
355 | local: Local, |
356 | layout: TyAndLayout<'tcx>, | |
357 | ) -> CPlace<'tcx> { | |
358 | let var1 = Variable::with_u32(fx.next_ssa_var); | |
359 | fx.next_ssa_var += 1; | |
360 | let var2 = Variable::with_u32(fx.next_ssa_var); | |
361 | fx.next_ssa_var += 1; | |
362 | ||
363 | let (ty1, ty2) = fx.clif_pair_type(layout.ty).unwrap(); | |
364 | fx.bcx.declare_var(var1, ty1); | |
365 | fx.bcx.declare_var(var2, ty2); | |
6a06907d | 366 | CPlace { inner: CPlaceInner::VarPair(local, var1, var2), layout } |
29967ef6 XL |
367 | } |
368 | ||
369 | pub(crate) fn for_ptr(ptr: Pointer, layout: TyAndLayout<'tcx>) -> CPlace<'tcx> { | |
6a06907d | 370 | CPlace { inner: CPlaceInner::Addr(ptr, None), layout } |
29967ef6 XL |
371 | } |
372 | ||
373 | pub(crate) fn for_ptr_with_extra( | |
374 | ptr: Pointer, | |
375 | extra: Value, | |
376 | layout: TyAndLayout<'tcx>, | |
377 | ) -> CPlace<'tcx> { | |
6a06907d | 378 | CPlace { inner: CPlaceInner::Addr(ptr, Some(extra)), layout } |
29967ef6 XL |
379 | } |
380 | ||
6a06907d | 381 | pub(crate) fn to_cvalue(self, fx: &mut FunctionCx<'_, '_, 'tcx>) -> CValue<'tcx> { |
29967ef6 XL |
382 | let layout = self.layout(); |
383 | match self.inner { | |
384 | CPlaceInner::Var(_local, var) => { | |
385 | let val = fx.bcx.use_var(var); | |
6a06907d | 386 | //fx.bcx.set_val_label(val, cranelift_codegen::ir::ValueLabel::new(var.index())); |
29967ef6 XL |
387 | CValue::by_val(val, layout) |
388 | } | |
389 | CPlaceInner::VarPair(_local, var1, var2) => { | |
390 | let val1 = fx.bcx.use_var(var1); | |
6a06907d | 391 | //fx.bcx.set_val_label(val1, cranelift_codegen::ir::ValueLabel::new(var1.index())); |
29967ef6 | 392 | let val2 = fx.bcx.use_var(var2); |
6a06907d | 393 | //fx.bcx.set_val_label(val2, cranelift_codegen::ir::ValueLabel::new(var2.index())); |
29967ef6 XL |
394 | CValue::by_val_pair(val1, val2, layout) |
395 | } | |
396 | CPlaceInner::VarLane(_local, var, lane) => { | |
397 | let val = fx.bcx.use_var(var); | |
6a06907d | 398 | //fx.bcx.set_val_label(val, cranelift_codegen::ir::ValueLabel::new(var.index())); |
29967ef6 XL |
399 | let val = fx.bcx.ins().extractlane(val, lane); |
400 | CValue::by_val(val, layout) | |
401 | } | |
402 | CPlaceInner::Addr(ptr, extra) => { | |
403 | if let Some(extra) = extra { | |
404 | CValue::by_ref_unsized(ptr, extra, layout) | |
405 | } else { | |
406 | CValue::by_ref(ptr, layout) | |
407 | } | |
408 | } | |
409 | } | |
410 | } | |
411 | ||
412 | pub(crate) fn to_ptr(self) -> Pointer { | |
413 | match self.to_ptr_maybe_unsized() { | |
414 | (ptr, None) => ptr, | |
415 | (_, Some(_)) => bug!("Expected sized cplace, found {:?}", self), | |
416 | } | |
417 | } | |
418 | ||
419 | pub(crate) fn to_ptr_maybe_unsized(self) -> (Pointer, Option<Value>) { | |
420 | match self.inner { | |
421 | CPlaceInner::Addr(ptr, extra) => (ptr, extra), | |
422 | CPlaceInner::Var(_, _) | |
423 | | CPlaceInner::VarPair(_, _, _) | |
424 | | CPlaceInner::VarLane(_, _, _) => bug!("Expected CPlace::Addr, found {:?}", self), | |
425 | } | |
426 | } | |
427 | ||
6a06907d | 428 | pub(crate) fn write_cvalue(self, fx: &mut FunctionCx<'_, '_, 'tcx>, from: CValue<'tcx>) { |
064997fb | 429 | assert_assignable(fx, from.layout().ty, self.layout().ty, 16); |
29967ef6 XL |
430 | |
431 | self.write_cvalue_maybe_transmute(fx, from, "write_cvalue"); | |
432 | } | |
433 | ||
434 | pub(crate) fn write_cvalue_transmute( | |
435 | self, | |
6a06907d | 436 | fx: &mut FunctionCx<'_, '_, 'tcx>, |
29967ef6 XL |
437 | from: CValue<'tcx>, |
438 | ) { | |
439 | self.write_cvalue_maybe_transmute(fx, from, "write_cvalue_transmute"); | |
440 | } | |
441 | ||
442 | fn write_cvalue_maybe_transmute( | |
443 | self, | |
6a06907d | 444 | fx: &mut FunctionCx<'_, '_, 'tcx>, |
29967ef6 | 445 | from: CValue<'tcx>, |
cdc7bbd5 | 446 | method: &'static str, |
29967ef6 XL |
447 | ) { |
448 | fn transmute_value<'tcx>( | |
6a06907d | 449 | fx: &mut FunctionCx<'_, '_, 'tcx>, |
29967ef6 XL |
450 | var: Variable, |
451 | data: Value, | |
452 | dst_ty: Type, | |
453 | ) { | |
454 | let src_ty = fx.bcx.func.dfg.value_type(data); | |
455 | assert_eq!( | |
456 | src_ty.bytes(), | |
457 | dst_ty.bytes(), | |
458 | "write_cvalue_transmute: {:?} -> {:?}", | |
459 | src_ty, | |
460 | dst_ty, | |
461 | ); | |
462 | let data = match (src_ty, dst_ty) { | |
463 | (_, _) if src_ty == dst_ty => data, | |
464 | ||
465 | // This is a `write_cvalue_transmute`. | |
466 | (types::I32, types::F32) | |
467 | | (types::F32, types::I32) | |
468 | | (types::I64, types::F64) | |
469 | | (types::F64, types::I64) => fx.bcx.ins().bitcast(dst_ty, data), | |
470 | _ if src_ty.is_vector() && dst_ty.is_vector() => { | |
471 | fx.bcx.ins().raw_bitcast(dst_ty, data) | |
472 | } | |
473 | _ if src_ty.is_vector() || dst_ty.is_vector() => { | |
474 | // FIXME do something more efficient for transmutes between vectors and integers. | |
475 | let stack_slot = fx.bcx.create_stack_slot(StackSlotData { | |
476 | kind: StackSlotKind::ExplicitSlot, | |
5869c6ff XL |
477 | // FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to |
478 | // specify stack slot alignment. | |
479 | size: (src_ty.bytes() + 15) / 16 * 16, | |
29967ef6 XL |
480 | }); |
481 | let ptr = Pointer::stack_slot(stack_slot); | |
482 | ptr.store(fx, data, MemFlags::trusted()); | |
483 | ptr.load(fx, dst_ty, MemFlags::trusted()) | |
484 | } | |
136023e0 XL |
485 | |
486 | // `CValue`s should never contain SSA-only types, so if you ended | |
487 | // up here having seen an error like `B1 -> I8`, then before | |
488 | // calling `write_cvalue` you need to add a `bint` instruction. | |
29967ef6 XL |
489 | _ => unreachable!("write_cvalue_transmute: {:?} -> {:?}", src_ty, dst_ty), |
490 | }; | |
6a06907d | 491 | //fx.bcx.set_val_label(data, cranelift_codegen::ir::ValueLabel::new(var.index())); |
29967ef6 XL |
492 | fx.bcx.def_var(var, data); |
493 | } | |
494 | ||
495 | assert_eq!(self.layout().size, from.layout().size); | |
496 | ||
cdc7bbd5 | 497 | if fx.clif_comments.enabled() { |
29967ef6 XL |
498 | use cranelift_codegen::cursor::{Cursor, CursorPosition}; |
499 | let cur_block = match fx.bcx.cursor().position() { | |
500 | CursorPosition::After(block) => block, | |
501 | _ => unreachable!(), | |
502 | }; | |
503 | fx.add_comment( | |
504 | fx.bcx.func.layout.last_inst(cur_block).unwrap(), | |
505 | format!( | |
506 | "{}: {:?}: {:?} <- {:?}: {:?}", | |
507 | method, | |
508 | self.inner(), | |
509 | self.layout().ty, | |
510 | from.0, | |
511 | from.layout().ty | |
512 | ), | |
513 | ); | |
514 | } | |
515 | ||
516 | let dst_layout = self.layout(); | |
517 | let to_ptr = match self.inner { | |
518 | CPlaceInner::Var(_local, var) => { | |
a2a8927a XL |
519 | if let ty::Array(element, len) = dst_layout.ty.kind() { |
520 | // Can only happen for vector types | |
521 | let len = | |
522 | u16::try_from(len.eval_usize(fx.tcx, ParamEnv::reveal_all())).unwrap(); | |
5099ac24 | 523 | let vector_ty = fx.clif_type(*element).unwrap().by(len).unwrap(); |
a2a8927a XL |
524 | |
525 | let data = match from.0 { | |
526 | CValueInner::ByRef(ptr, None) => { | |
527 | let mut flags = MemFlags::new(); | |
528 | flags.set_notrap(); | |
529 | ptr.load(fx, vector_ty, flags) | |
530 | } | |
531 | CValueInner::ByVal(_) | |
532 | | CValueInner::ByValPair(_, _) | |
533 | | CValueInner::ByRef(_, Some(_)) => bug!("array should be ByRef"), | |
534 | }; | |
535 | ||
536 | fx.bcx.def_var(var, data); | |
537 | return; | |
538 | } | |
29967ef6 XL |
539 | let data = CValue(from.0, dst_layout).load_scalar(fx); |
540 | let dst_ty = fx.clif_type(self.layout().ty).unwrap(); | |
541 | transmute_value(fx, var, data, dst_ty); | |
542 | return; | |
543 | } | |
544 | CPlaceInner::VarPair(_local, var1, var2) => { | |
545 | let (data1, data2) = CValue(from.0, dst_layout).load_scalar_pair(fx); | |
546 | let (dst_ty1, dst_ty2) = fx.clif_pair_type(self.layout().ty).unwrap(); | |
547 | transmute_value(fx, var1, data1, dst_ty1); | |
548 | transmute_value(fx, var2, data2, dst_ty2); | |
549 | return; | |
550 | } | |
551 | CPlaceInner::VarLane(_local, var, lane) => { | |
552 | let data = from.load_scalar(fx); | |
553 | ||
554 | // First get the old vector | |
555 | let vector = fx.bcx.use_var(var); | |
6a06907d | 556 | //fx.bcx.set_val_label(vector, cranelift_codegen::ir::ValueLabel::new(var.index())); |
29967ef6 XL |
557 | |
558 | // Next insert the written lane into the vector | |
559 | let vector = fx.bcx.ins().insertlane(vector, data, lane); | |
560 | ||
561 | // Finally write the new vector | |
6a06907d | 562 | //fx.bcx.set_val_label(vector, cranelift_codegen::ir::ValueLabel::new(var.index())); |
29967ef6 XL |
563 | fx.bcx.def_var(var, vector); |
564 | ||
565 | return; | |
566 | } | |
567 | CPlaceInner::Addr(ptr, None) => { | |
568 | if dst_layout.size == Size::ZERO || dst_layout.abi == Abi::Uninhabited { | |
569 | return; | |
570 | } | |
571 | ptr | |
572 | } | |
573 | CPlaceInner::Addr(_, Some(_)) => bug!("Can't write value to unsized place {:?}", self), | |
574 | }; | |
575 | ||
576 | let mut flags = MemFlags::new(); | |
577 | flags.set_notrap(); | |
578 | match from.layout().abi { | |
579 | // FIXME make Abi::Vector work too | |
580 | Abi::Scalar(_) => { | |
581 | let val = from.load_scalar(fx); | |
582 | to_ptr.store(fx, val, flags); | |
583 | return; | |
584 | } | |
c295e0f8 | 585 | Abi::ScalarPair(a_scalar, b_scalar) => { |
29967ef6 XL |
586 | let (value, extra) = from.load_scalar_pair(fx); |
587 | let b_offset = scalar_pair_calculate_b_offset(fx.tcx, a_scalar, b_scalar); | |
588 | to_ptr.store(fx, value, flags); | |
589 | to_ptr.offset(fx, b_offset).store(fx, extra, flags); | |
590 | return; | |
591 | } | |
592 | _ => {} | |
593 | } | |
594 | ||
595 | match from.0 { | |
596 | CValueInner::ByVal(val) => { | |
597 | to_ptr.store(fx, val, flags); | |
598 | } | |
599 | CValueInner::ByValPair(_, _) => { | |
6a06907d | 600 | bug!("Non ScalarPair abi {:?} for ByValPair CValue", dst_layout.abi); |
29967ef6 XL |
601 | } |
602 | CValueInner::ByRef(from_ptr, None) => { | |
603 | let from_addr = from_ptr.get_addr(fx); | |
604 | let to_addr = to_ptr.get_addr(fx); | |
605 | let src_layout = from.1; | |
606 | let size = dst_layout.size.bytes(); | |
607 | let src_align = src_layout.align.abi.bytes() as u8; | |
608 | let dst_align = dst_layout.align.abi.bytes() as u8; | |
609 | fx.bcx.emit_small_memory_copy( | |
a2a8927a | 610 | fx.target_config, |
29967ef6 XL |
611 | to_addr, |
612 | from_addr, | |
613 | size, | |
614 | dst_align, | |
615 | src_align, | |
616 | true, | |
17df50a5 | 617 | MemFlags::trusted(), |
29967ef6 XL |
618 | ); |
619 | } | |
620 | CValueInner::ByRef(_, Some(_)) => todo!(), | |
621 | } | |
622 | } | |
623 | ||
624 | pub(crate) fn place_field( | |
625 | self, | |
6a06907d | 626 | fx: &mut FunctionCx<'_, '_, 'tcx>, |
29967ef6 XL |
627 | field: mir::Field, |
628 | ) -> CPlace<'tcx> { | |
629 | let layout = self.layout(); | |
630 | ||
631 | match self.inner { | |
a2a8927a XL |
632 | CPlaceInner::Var(local, var) => match layout.ty.kind() { |
633 | ty::Array(_, _) => { | |
634 | // Can only happen for vector types | |
29967ef6 XL |
635 | return CPlace { |
636 | inner: CPlaceInner::VarLane(local, var, field.as_u32().try_into().unwrap()), | |
637 | layout: layout.field(fx, field.as_u32().try_into().unwrap()), | |
638 | }; | |
639 | } | |
a2a8927a XL |
640 | ty::Adt(adt_def, substs) if layout.ty.is_simd() => { |
641 | let f0_ty = adt_def.non_enum_variant().fields[0].ty(fx.tcx, substs); | |
642 | ||
643 | match f0_ty.kind() { | |
644 | ty::Array(_, _) => { | |
645 | assert_eq!(field.as_u32(), 0); | |
646 | return CPlace { | |
647 | inner: CPlaceInner::Var(local, var), | |
648 | layout: layout.field(fx, field.as_u32().try_into().unwrap()), | |
649 | }; | |
650 | } | |
651 | _ => { | |
652 | return CPlace { | |
653 | inner: CPlaceInner::VarLane( | |
654 | local, | |
655 | var, | |
656 | field.as_u32().try_into().unwrap(), | |
657 | ), | |
658 | layout: layout.field(fx, field.as_u32().try_into().unwrap()), | |
659 | }; | |
660 | } | |
661 | } | |
662 | } | |
663 | _ => {} | |
664 | }, | |
29967ef6 XL |
665 | CPlaceInner::VarPair(local, var1, var2) => { |
666 | let layout = layout.field(&*fx, field.index()); | |
667 | ||
668 | match field.as_u32() { | |
6a06907d XL |
669 | 0 => return CPlace { inner: CPlaceInner::Var(local, var1), layout }, |
670 | 1 => return CPlace { inner: CPlaceInner::Var(local, var2), layout }, | |
29967ef6 XL |
671 | _ => unreachable!("field should be 0 or 1"), |
672 | } | |
673 | } | |
674 | _ => {} | |
675 | } | |
676 | ||
677 | let (base, extra) = self.to_ptr_maybe_unsized(); | |
678 | ||
679 | let (field_ptr, field_layout) = codegen_field(fx, base, extra, layout, field); | |
680 | if field_layout.is_unsized() { | |
a2a8927a XL |
681 | if let ty::Foreign(_) = field_layout.ty.kind() { |
682 | assert!(extra.is_none()); | |
683 | CPlace::for_ptr(field_ptr, field_layout) | |
684 | } else { | |
685 | CPlace::for_ptr_with_extra(field_ptr, extra.unwrap(), field_layout) | |
686 | } | |
29967ef6 XL |
687 | } else { |
688 | CPlace::for_ptr(field_ptr, field_layout) | |
689 | } | |
690 | } | |
691 | ||
94222f64 XL |
692 | /// Like [`CPlace::place_field`] except handling ADTs containing a single array field in a way |
693 | /// such that you can access individual lanes. | |
694 | pub(crate) fn place_lane( | |
695 | self, | |
696 | fx: &mut FunctionCx<'_, '_, 'tcx>, | |
697 | lane_idx: u64, | |
698 | ) -> CPlace<'tcx> { | |
699 | let layout = self.layout(); | |
700 | assert!(layout.ty.is_simd()); | |
701 | let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx); | |
702 | let lane_layout = fx.layout_of(lane_ty); | |
703 | assert!(lane_idx < lane_count); | |
704 | ||
705 | match self.inner { | |
706 | CPlaceInner::Var(local, var) => { | |
707 | assert!(matches!(layout.abi, Abi::Vector { .. })); | |
708 | CPlace { | |
709 | inner: CPlaceInner::VarLane(local, var, lane_idx.try_into().unwrap()), | |
710 | layout: lane_layout, | |
711 | } | |
712 | } | |
713 | CPlaceInner::VarPair(_, _, _) => unreachable!(), | |
714 | CPlaceInner::VarLane(_, _, _) => unreachable!(), | |
715 | CPlaceInner::Addr(ptr, None) => { | |
716 | let field_offset = lane_layout.size * lane_idx; | |
717 | let field_ptr = ptr.offset_i64(fx, i64::try_from(field_offset.bytes()).unwrap()); | |
718 | CPlace::for_ptr(field_ptr, lane_layout) | |
719 | } | |
720 | CPlaceInner::Addr(_, Some(_)) => unreachable!(), | |
721 | } | |
722 | } | |
723 | ||
29967ef6 XL |
724 | pub(crate) fn place_index( |
725 | self, | |
6a06907d | 726 | fx: &mut FunctionCx<'_, '_, 'tcx>, |
29967ef6 XL |
727 | index: Value, |
728 | ) -> CPlace<'tcx> { | |
729 | let (elem_layout, ptr) = match self.layout().ty.kind() { | |
5099ac24 FG |
730 | ty::Array(elem_ty, _) => (fx.layout_of(*elem_ty), self.to_ptr()), |
731 | ty::Slice(elem_ty) => (fx.layout_of(*elem_ty), self.to_ptr_maybe_unsized().0), | |
29967ef6 XL |
732 | _ => bug!("place_index({:?})", self.layout().ty), |
733 | }; | |
734 | ||
6a06907d | 735 | let offset = fx.bcx.ins().imul_imm(index, elem_layout.size.bytes() as i64); |
29967ef6 XL |
736 | |
737 | CPlace::for_ptr(ptr.offset_value(fx, offset), elem_layout) | |
738 | } | |
739 | ||
6a06907d | 740 | pub(crate) fn place_deref(self, fx: &mut FunctionCx<'_, '_, 'tcx>) -> CPlace<'tcx> { |
29967ef6 XL |
741 | let inner_layout = fx.layout_of(self.layout().ty.builtin_deref(true).unwrap().ty); |
742 | if has_ptr_meta(fx.tcx, inner_layout.ty) { | |
743 | let (addr, extra) = self.to_cvalue(fx).load_scalar_pair(fx); | |
744 | CPlace::for_ptr_with_extra(Pointer::new(addr), extra, inner_layout) | |
745 | } else { | |
6a06907d | 746 | CPlace::for_ptr(Pointer::new(self.to_cvalue(fx).load_scalar(fx)), inner_layout) |
29967ef6 XL |
747 | } |
748 | } | |
749 | ||
750 | pub(crate) fn place_ref( | |
751 | self, | |
6a06907d | 752 | fx: &mut FunctionCx<'_, '_, 'tcx>, |
29967ef6 XL |
753 | layout: TyAndLayout<'tcx>, |
754 | ) -> CValue<'tcx> { | |
755 | if has_ptr_meta(fx.tcx, self.layout().ty) { | |
756 | let (ptr, extra) = self.to_ptr_maybe_unsized(); | |
757 | CValue::by_val_pair( | |
758 | ptr.get_addr(fx), | |
759 | extra.expect("unsized type without metadata"), | |
760 | layout, | |
761 | ) | |
762 | } else { | |
763 | CValue::by_val(self.to_ptr().get_addr(fx), layout) | |
764 | } | |
765 | } | |
766 | ||
767 | pub(crate) fn downcast_variant( | |
768 | self, | |
6a06907d | 769 | fx: &FunctionCx<'_, '_, 'tcx>, |
29967ef6 XL |
770 | variant: VariantIdx, |
771 | ) -> Self { | |
772 | assert!(!self.layout().is_unsized()); | |
773 | let layout = self.layout().for_variant(fx, variant); | |
6a06907d | 774 | CPlace { inner: self.inner, layout } |
29967ef6 XL |
775 | } |
776 | } | |
5869c6ff XL |
777 | |
778 | #[track_caller] | |
779 | pub(crate) fn assert_assignable<'tcx>( | |
6a06907d | 780 | fx: &FunctionCx<'_, '_, 'tcx>, |
5869c6ff XL |
781 | from_ty: Ty<'tcx>, |
782 | to_ty: Ty<'tcx>, | |
064997fb | 783 | limit: usize, |
5869c6ff | 784 | ) { |
064997fb FG |
785 | if limit == 0 { |
786 | // assert_assignable exists solely to catch bugs in cg_clif. it isn't necessary for | |
787 | // soundness. don't attempt to check deep types to avoid exponential behavior in certain | |
788 | // cases. | |
789 | return; | |
790 | } | |
5869c6ff XL |
791 | match (from_ty.kind(), to_ty.kind()) { |
792 | (ty::Ref(_, a, _), ty::Ref(_, b, _)) | |
793 | | ( | |
794 | ty::RawPtr(TypeAndMut { ty: a, mutbl: _ }), | |
795 | ty::RawPtr(TypeAndMut { ty: b, mutbl: _ }), | |
796 | ) => { | |
064997fb | 797 | assert_assignable(fx, *a, *b, limit - 1); |
5869c6ff XL |
798 | } |
799 | (ty::Ref(_, a, _), ty::RawPtr(TypeAndMut { ty: b, mutbl: _ })) | |
800 | | (ty::RawPtr(TypeAndMut { ty: a, mutbl: _ }), ty::Ref(_, b, _)) => { | |
064997fb | 801 | assert_assignable(fx, *a, *b, limit - 1); |
5869c6ff XL |
802 | } |
803 | (ty::FnPtr(_), ty::FnPtr(_)) => { | |
804 | let from_sig = fx.tcx.normalize_erasing_late_bound_regions( | |
805 | ParamEnv::reveal_all(), | |
806 | from_ty.fn_sig(fx.tcx), | |
807 | ); | |
808 | let to_sig = fx | |
809 | .tcx | |
810 | .normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), to_ty.fn_sig(fx.tcx)); | |
811 | assert_eq!( | |
812 | from_sig, to_sig, | |
813 | "Can't write fn ptr with incompatible sig {:?} to place with sig {:?}\n\n{:#?}", | |
814 | from_sig, to_sig, fx, | |
815 | ); | |
816 | // fn(&T) -> for<'l> fn(&'l T) is allowed | |
817 | } | |
818 | (&ty::Dynamic(from_traits, _), &ty::Dynamic(to_traits, _)) => { | |
819 | for (from, to) in from_traits.iter().zip(to_traits) { | |
6a06907d XL |
820 | let from = |
821 | fx.tcx.normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), from); | |
822 | let to = fx.tcx.normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), to); | |
5869c6ff XL |
823 | assert_eq!( |
824 | from, to, | |
825 | "Can't write trait object of incompatible traits {:?} to place with traits {:?}\n\n{:#?}", | |
826 | from_traits, to_traits, fx, | |
827 | ); | |
828 | } | |
829 | // dyn for<'r> Trait<'r> -> dyn Trait<'_> is allowed | |
830 | } | |
064997fb FG |
831 | (&ty::Tuple(types_a), &ty::Tuple(types_b)) => { |
832 | let mut types_a = types_a.iter(); | |
833 | let mut types_b = types_b.iter(); | |
834 | loop { | |
835 | match (types_a.next(), types_b.next()) { | |
836 | (Some(a), Some(b)) => assert_assignable(fx, a, b, limit - 1), | |
837 | (None, None) => return, | |
838 | (Some(_), None) | (None, Some(_)) => panic!("{:#?}/{:#?}", from_ty, to_ty), | |
839 | } | |
840 | } | |
841 | } | |
cdc7bbd5 | 842 | (&ty::Adt(adt_def_a, substs_a), &ty::Adt(adt_def_b, substs_b)) |
5e7ed085 | 843 | if adt_def_a.did() == adt_def_b.did() => |
cdc7bbd5 XL |
844 | { |
845 | let mut types_a = substs_a.types(); | |
846 | let mut types_b = substs_b.types(); | |
847 | loop { | |
848 | match (types_a.next(), types_b.next()) { | |
064997fb | 849 | (Some(a), Some(b)) => assert_assignable(fx, a, b, limit - 1), |
cdc7bbd5 XL |
850 | (None, None) => return, |
851 | (Some(_), None) | (None, Some(_)) => panic!("{:#?}/{:#?}", from_ty, to_ty), | |
852 | } | |
853 | } | |
854 | } | |
064997fb FG |
855 | (ty::Array(a, _), ty::Array(b, _)) => assert_assignable(fx, *a, *b, limit - 1), |
856 | (&ty::Closure(def_id_a, substs_a), &ty::Closure(def_id_b, substs_b)) | |
857 | if def_id_a == def_id_b => | |
858 | { | |
859 | let mut types_a = substs_a.types(); | |
860 | let mut types_b = substs_b.types(); | |
861 | loop { | |
862 | match (types_a.next(), types_b.next()) { | |
863 | (Some(a), Some(b)) => assert_assignable(fx, a, b, limit - 1), | |
864 | (None, None) => return, | |
865 | (Some(_), None) | (None, Some(_)) => panic!("{:#?}/{:#?}", from_ty, to_ty), | |
866 | } | |
867 | } | |
868 | } | |
869 | (ty::Param(_), _) | (_, ty::Param(_)) if fx.tcx.sess.opts.unstable_opts.polymorphize => { | |
870 | // No way to check if it is correct or not with polymorphization enabled | |
871 | } | |
5869c6ff XL |
872 | _ => { |
873 | assert_eq!( | |
064997fb FG |
874 | from_ty, |
875 | to_ty, | |
5869c6ff | 876 | "Can't write value with incompatible type {:?} to place with type {:?}\n\n{:#?}", |
064997fb FG |
877 | from_ty.kind(), |
878 | to_ty.kind(), | |
879 | fx, | |
5869c6ff XL |
880 | ); |
881 | } | |
882 | } | |
883 | } |