]>
Commit | Line | Data |
---|---|---|
29967ef6 XL |
1 | //! Codegen of the [`PointerCast::Unsize`] operation. |
2 | //! | |
3 | //! [`PointerCast::Unsize`]: `rustc_middle::ty::adjustment::PointerCast::Unsize` | |
4 | ||
5 | use crate::prelude::*; | |
6 | ||
7 | // Adapted from https://github.com/rust-lang/rust/blob/2a663555ddf36f6b041445894a8c175cd1bc718c/src/librustc_codegen_ssa/base.rs#L159-L307 | |
8 | ||
9 | /// Retrieve the information we are losing (making dynamic) in an unsizing | |
10 | /// adjustment. | |
11 | /// | |
12 | /// The `old_info` argument is a bit funny. It is intended for use | |
13 | /// in an upcast, where the new vtable for an object will be derived | |
14 | /// from the old one. | |
15 | pub(crate) fn unsized_info<'tcx>( | |
6a06907d | 16 | fx: &mut FunctionCx<'_, '_, 'tcx>, |
29967ef6 XL |
17 | source: Ty<'tcx>, |
18 | target: Ty<'tcx>, | |
19 | old_info: Option<Value>, | |
20 | ) -> Value { | |
21 | let (source, target) = | |
6a06907d | 22 | fx.tcx.struct_lockstep_tails_erasing_lifetimes(source, target, ParamEnv::reveal_all()); |
29967ef6 | 23 | match (&source.kind(), &target.kind()) { |
6a06907d XL |
24 | (&ty::Array(_, len), &ty::Slice(_)) => fx |
25 | .bcx | |
26 | .ins() | |
27 | .iconst(fx.pointer_type, len.eval_usize(fx.tcx, ParamEnv::reveal_all()) as i64), | |
2b03887a FG |
28 | ( |
29 | &ty::Dynamic(ref data_a, _, src_dyn_kind), | |
30 | &ty::Dynamic(ref data_b, _, target_dyn_kind), | |
31 | ) => { | |
32 | assert_eq!(src_dyn_kind, target_dyn_kind); | |
33 | ||
94222f64 XL |
34 | let old_info = |
35 | old_info.expect("unsized_info: missing old info for trait upcasting coercion"); | |
36 | if data_a.principal_def_id() == data_b.principal_def_id() { | |
f2b60f7d | 37 | // A NOP cast that doesn't actually change anything, should be allowed even with invalid vtables. |
94222f64 XL |
38 | return old_info; |
39 | } | |
40 | ||
41 | // trait upcasting coercion | |
42 | let vptr_entry_idx = | |
43 | fx.tcx.vtable_trait_upcasting_coercion_new_vptr_slot((source, target)); | |
44 | ||
45 | if let Some(entry_idx) = vptr_entry_idx { | |
46 | let entry_idx = u32::try_from(entry_idx).unwrap(); | |
47 | let entry_offset = entry_idx * fx.pointer_type.bytes(); | |
48 | let vptr_ptr = Pointer::new(old_info).offset_i64(fx, entry_offset.into()).load( | |
49 | fx, | |
50 | fx.pointer_type, | |
51 | crate::vtable::vtable_memflags(), | |
52 | ); | |
53 | vptr_ptr | |
54 | } else { | |
55 | old_info | |
56 | } | |
29967ef6 | 57 | } |
136023e0 | 58 | (_, &ty::Dynamic(ref data, ..)) => crate::vtable::get_vtable(fx, source, data.principal()), |
6a06907d | 59 | _ => bug!("unsized_info: invalid unsizing {:?} -> {:?}", source, target), |
29967ef6 XL |
60 | } |
61 | } | |
62 | ||
94222f64 XL |
63 | /// Coerce `src` to `dst_ty`. |
64 | fn unsize_ptr<'tcx>( | |
6a06907d | 65 | fx: &mut FunctionCx<'_, '_, 'tcx>, |
29967ef6 XL |
66 | src: Value, |
67 | src_layout: TyAndLayout<'tcx>, | |
68 | dst_layout: TyAndLayout<'tcx>, | |
94222f64 | 69 | old_info: Option<Value>, |
29967ef6 XL |
70 | ) -> (Value, Value) { |
71 | match (&src_layout.ty.kind(), &dst_layout.ty.kind()) { | |
72 | (&ty::Ref(_, a, _), &ty::Ref(_, b, _)) | |
73 | | (&ty::Ref(_, a, _), &ty::RawPtr(ty::TypeAndMut { ty: b, .. })) | |
74 | | (&ty::RawPtr(ty::TypeAndMut { ty: a, .. }), &ty::RawPtr(ty::TypeAndMut { ty: b, .. })) => { | |
5099ac24 | 75 | (src, unsized_info(fx, *a, *b, old_info)) |
29967ef6 XL |
76 | } |
77 | (&ty::Adt(def_a, _), &ty::Adt(def_b, _)) if def_a.is_box() && def_b.is_box() => { | |
78 | let (a, b) = (src_layout.ty.boxed_ty(), dst_layout.ty.boxed_ty()); | |
94222f64 | 79 | (src, unsized_info(fx, a, b, old_info)) |
29967ef6 XL |
80 | } |
81 | (&ty::Adt(def_a, _), &ty::Adt(def_b, _)) => { | |
82 | assert_eq!(def_a, def_b); | |
83 | ||
94222f64 XL |
84 | if src_layout == dst_layout { |
85 | return (src, old_info.unwrap()); | |
86 | } | |
87 | ||
29967ef6 XL |
88 | let mut result = None; |
89 | for i in 0..src_layout.fields.count() { | |
90 | let src_f = src_layout.field(fx, i); | |
91 | assert_eq!(src_layout.fields.offset(i).bytes(), 0); | |
92 | assert_eq!(dst_layout.fields.offset(i).bytes(), 0); | |
93 | if src_f.is_zst() { | |
94 | continue; | |
95 | } | |
96 | assert_eq!(src_layout.size, src_f.size); | |
97 | ||
98 | let dst_f = dst_layout.field(fx, i); | |
99 | assert_ne!(src_f.ty, dst_f.ty); | |
100 | assert_eq!(result, None); | |
94222f64 | 101 | result = Some(unsize_ptr(fx, src, src_f, dst_f, old_info)); |
29967ef6 XL |
102 | } |
103 | result.unwrap() | |
104 | } | |
94222f64 | 105 | _ => bug!("unsize_ptr: called on bad types"), |
29967ef6 XL |
106 | } |
107 | } | |
108 | ||
2b03887a FG |
109 | /// Coerces `src` to `dst_ty` which is guaranteed to be a `dyn*` type. |
110 | pub(crate) fn cast_to_dyn_star<'tcx>( | |
111 | fx: &mut FunctionCx<'_, '_, 'tcx>, | |
112 | src: Value, | |
113 | src_ty_and_layout: TyAndLayout<'tcx>, | |
114 | dst_ty: Ty<'tcx>, | |
115 | old_info: Option<Value>, | |
116 | ) -> (Value, Value) { | |
117 | assert!( | |
118 | matches!(dst_ty.kind(), ty::Dynamic(_, _, ty::DynStar)), | |
119 | "destination type must be a dyn*" | |
120 | ); | |
121 | (src, unsized_info(fx, src_ty_and_layout.ty, dst_ty, old_info)) | |
122 | } | |
123 | ||
29967ef6 XL |
124 | /// Coerce `src`, which is a reference to a value of type `src_ty`, |
125 | /// to a value of type `dst_ty` and store the result in `dst` | |
126 | pub(crate) fn coerce_unsized_into<'tcx>( | |
6a06907d | 127 | fx: &mut FunctionCx<'_, '_, 'tcx>, |
29967ef6 XL |
128 | src: CValue<'tcx>, |
129 | dst: CPlace<'tcx>, | |
130 | ) { | |
131 | let src_ty = src.layout().ty; | |
132 | let dst_ty = dst.layout().ty; | |
133 | let mut coerce_ptr = || { | |
6a06907d XL |
134 | let (base, info) = |
135 | if fx.layout_of(src.layout().ty.builtin_deref(true).unwrap().ty).is_unsized() { | |
94222f64 XL |
136 | let (old_base, old_info) = src.load_scalar_pair(fx); |
137 | unsize_ptr(fx, old_base, src.layout(), dst.layout(), Some(old_info)) | |
6a06907d XL |
138 | } else { |
139 | let base = src.load_scalar(fx); | |
94222f64 | 140 | unsize_ptr(fx, base, src.layout(), dst.layout(), None) |
6a06907d | 141 | }; |
29967ef6 XL |
142 | dst.write_cvalue(fx, CValue::by_val_pair(base, info, dst.layout())); |
143 | }; | |
144 | match (&src_ty.kind(), &dst_ty.kind()) { | |
145 | (&ty::Ref(..), &ty::Ref(..)) | |
146 | | (&ty::Ref(..), &ty::RawPtr(..)) | |
147 | | (&ty::RawPtr(..), &ty::RawPtr(..)) => coerce_ptr(), | |
148 | (&ty::Adt(def_a, _), &ty::Adt(def_b, _)) => { | |
149 | assert_eq!(def_a, def_b); | |
150 | ||
5e7ed085 | 151 | for i in 0..def_a.variant(VariantIdx::new(0)).fields.len() { |
29967ef6 XL |
152 | let src_f = src.value_field(fx, mir::Field::new(i)); |
153 | let dst_f = dst.place_field(fx, mir::Field::new(i)); | |
154 | ||
155 | if dst_f.layout().is_zst() { | |
156 | continue; | |
157 | } | |
158 | ||
159 | if src_f.layout().ty == dst_f.layout().ty { | |
160 | dst_f.write_cvalue(fx, src_f); | |
161 | } else { | |
162 | coerce_unsized_into(fx, src_f, dst_f); | |
163 | } | |
164 | } | |
165 | } | |
6a06907d | 166 | _ => bug!("coerce_unsized_into: invalid coercion {:?} -> {:?}", src_ty, dst_ty), |
29967ef6 XL |
167 | } |
168 | } | |
169 | ||
2b03887a FG |
170 | pub(crate) fn coerce_dyn_star<'tcx>( |
171 | fx: &mut FunctionCx<'_, '_, 'tcx>, | |
172 | src: CValue<'tcx>, | |
173 | dst: CPlace<'tcx>, | |
174 | ) { | |
175 | let (data, extra) = if let ty::Dynamic(_, _, ty::DynStar) = src.layout().ty.kind() { | |
176 | let (data, vtable) = src.load_scalar_pair(fx); | |
177 | (data, Some(vtable)) | |
178 | } else { | |
179 | let data = src.load_scalar(fx); | |
180 | (data, None) | |
181 | }; | |
182 | ||
183 | let (data, vtable) = cast_to_dyn_star(fx, data, src.layout(), dst.layout().ty, extra); | |
184 | ||
185 | dst.write_cvalue(fx, CValue::by_val_pair(data, vtable, dst.layout())); | |
186 | } | |
187 | ||
29967ef6 XL |
188 | // Adapted from https://github.com/rust-lang/rust/blob/2a663555ddf36f6b041445894a8c175cd1bc718c/src/librustc_codegen_ssa/glue.rs |
189 | ||
190 | pub(crate) fn size_and_align_of_dst<'tcx>( | |
6a06907d | 191 | fx: &mut FunctionCx<'_, '_, 'tcx>, |
29967ef6 XL |
192 | layout: TyAndLayout<'tcx>, |
193 | info: Value, | |
194 | ) -> (Value, Value) { | |
064997fb | 195 | assert!(layout.is_unsized() || layout.abi == Abi::Uninhabited); |
29967ef6 XL |
196 | match layout.ty.kind() { |
197 | ty::Dynamic(..) => { | |
198 | // load size/align from vtable | |
6a06907d | 199 | (crate::vtable::size_of_obj(fx, info), crate::vtable::min_align_of_obj(fx, info)) |
29967ef6 XL |
200 | } |
201 | ty::Slice(_) | ty::Str => { | |
202 | let unit = layout.field(fx, 0); | |
203 | // The info in this case is the length of the str, so the size is that | |
204 | // times the unit size. | |
205 | ( | |
206 | fx.bcx.ins().imul_imm(info, unit.size.bytes() as i64), | |
6a06907d | 207 | fx.bcx.ins().iconst(fx.pointer_type, unit.align.abi.bytes() as i64), |
29967ef6 XL |
208 | ) |
209 | } | |
210 | _ => { | |
211 | // First get the size of all statically known fields. | |
212 | // Don't use size_of because it also rounds up to alignment, which we | |
213 | // want to avoid, as the unsized field's alignment could be smaller. | |
214 | assert!(!layout.ty.is_simd()); | |
215 | ||
216 | let i = layout.fields.count() - 1; | |
217 | let sized_size = layout.fields.offset(i).bytes(); | |
218 | let sized_align = layout.align.abi.bytes(); | |
219 | let sized_align = fx.bcx.ins().iconst(fx.pointer_type, sized_align as i64); | |
220 | ||
221 | // Recurse to get the size of the dynamically sized field (must be | |
222 | // the last field). | |
223 | let field_layout = layout.field(fx, i); | |
224 | let (unsized_size, mut unsized_align) = size_and_align_of_dst(fx, field_layout, info); | |
225 | ||
226 | // FIXME (#26403, #27023): We should be adding padding | |
227 | // to `sized_size` (to accommodate the `unsized_align` | |
228 | // required of the unsized field that follows) before | |
229 | // summing it with `sized_size`. (Note that since #26403 | |
230 | // is unfixed, we do not yet add the necessary padding | |
231 | // here. But this is where the add would go.) | |
232 | ||
233 | // Return the sum of sizes and max of aligns. | |
234 | let size = fx.bcx.ins().iadd_imm(unsized_size, sized_size as i64); | |
235 | ||
236 | // Packed types ignore the alignment of their fields. | |
237 | if let ty::Adt(def, _) = layout.ty.kind() { | |
5e7ed085 | 238 | if def.repr().packed() { |
29967ef6 XL |
239 | unsized_align = sized_align; |
240 | } | |
241 | } | |
242 | ||
243 | // Choose max of two known alignments (combined value must | |
244 | // be aligned according to more restrictive of the two). | |
6a06907d | 245 | let cmp = fx.bcx.ins().icmp(IntCC::UnsignedGreaterThan, sized_align, unsized_align); |
29967ef6 XL |
246 | let align = fx.bcx.ins().select(cmp, sized_align, unsized_align); |
247 | ||
248 | // Issue #27023: must add any necessary padding to `size` | |
249 | // (to make it a multiple of `align`) before returning it. | |
250 | // | |
251 | // Namely, the returned size should be, in C notation: | |
252 | // | |
253 | // `size + ((size & (align-1)) ? align : 0)` | |
254 | // | |
255 | // emulated via the semi-standard fast bit trick: | |
256 | // | |
257 | // `(size + (align-1)) & -align` | |
258 | let addend = fx.bcx.ins().iadd_imm(align, -1); | |
259 | let add = fx.bcx.ins().iadd(size, addend); | |
260 | let neg = fx.bcx.ins().ineg(align); | |
261 | let size = fx.bcx.ins().band(add, neg); | |
262 | ||
263 | (size, align) | |
264 | } | |
265 | } | |
266 | } |