]>
Commit | Line | Data |
---|---|---|
60c5eb7d | 1 | use super::operand::OperandValue; |
dfeec247 | 2 | use super::{FunctionCx, LocalRef}; |
60c5eb7d | 3 | |
9fa01778 XL |
4 | use crate::common::IntPredicate; |
5 | use crate::glue; | |
9fa01778 | 6 | use crate::traits::*; |
ff7c6d11 | 7 | |
ba9703b0 XL |
8 | use rustc_middle::mir; |
9 | use rustc_middle::mir::tcx::PlaceTy; | |
c295e0f8 | 10 | use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf, TyAndLayout}; |
ba9703b0 | 11 | use rustc_middle::ty::{self, Ty}; |
f035d41b | 12 | use rustc_target::abi::{Abi, Align, FieldsShape, Int, TagEncoding}; |
c295e0f8 | 13 | use rustc_target::abi::{VariantIdx, Variants}; |
ff7c6d11 XL |
14 | |
15 | #[derive(Copy, Clone, Debug)] | |
a1dfa0c6 | 16 | pub struct PlaceRef<'tcx, V> { |
60c5eb7d | 17 | /// A pointer to the contents of the place. |
a1dfa0c6 | 18 | pub llval: V, |
ff7c6d11 | 19 | |
60c5eb7d | 20 | /// This place's extra data if it is unsized, or `None` if null. |
a1dfa0c6 | 21 | pub llextra: Option<V>, |
ff7c6d11 | 22 | |
60c5eb7d | 23 | /// The monomorphized type of this place, including variant information. |
ba9703b0 | 24 | pub layout: TyAndLayout<'tcx>, |
ff7c6d11 | 25 | |
60c5eb7d | 26 | /// The alignment we know for this place. |
ff7c6d11 XL |
27 | pub align: Align, |
28 | } | |
29 | ||
dc9dc135 | 30 | impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> { |
ba9703b0 | 31 | pub fn new_sized(llval: V, layout: TyAndLayout<'tcx>) -> PlaceRef<'tcx, V> { |
e1599b0c | 32 | assert!(!layout.is_unsized()); |
dfeec247 | 33 | PlaceRef { llval, llextra: None, layout, align: layout.align.abi } |
e1599b0c XL |
34 | } |
35 | ||
ba9703b0 XL |
36 | pub fn new_sized_aligned( |
37 | llval: V, | |
38 | layout: TyAndLayout<'tcx>, | |
39 | align: Align, | |
40 | ) -> PlaceRef<'tcx, V> { | |
b7449926 | 41 | assert!(!layout.is_unsized()); |
dfeec247 | 42 | PlaceRef { llval, llextra: None, layout, align } |
9fa01778 XL |
43 | } |
44 | ||
e74abb32 XL |
45 | // FIXME(eddyb) pass something else for the name so no work is done |
46 | // unless LLVM IR names are turned on (e.g. for `--emit=llvm-ir`). | |
a1dfa0c6 XL |
47 | pub fn alloca<Bx: BuilderMethods<'a, 'tcx, Value = V>>( |
48 | bx: &mut Bx, | |
ba9703b0 | 49 | layout: TyAndLayout<'tcx>, |
a1dfa0c6 | 50 | ) -> Self { |
b7449926 | 51 | assert!(!layout.is_unsized(), "tried to statically allocate unsized place"); |
e1599b0c XL |
52 | let tmp = bx.alloca(bx.cx().backend_type(layout), layout.align.abi); |
53 | Self::new_sized(tmp, layout) | |
ff7c6d11 XL |
54 | } |
55 | ||
b7449926 | 56 | /// Returns a place for an indirect reference to an unsized place. |
e74abb32 XL |
57 | // FIXME(eddyb) pass something else for the name so no work is done |
58 | // unless LLVM IR names are turned on (e.g. for `--emit=llvm-ir`). | |
a1dfa0c6 XL |
59 | pub fn alloca_unsized_indirect<Bx: BuilderMethods<'a, 'tcx, Value = V>>( |
60 | bx: &mut Bx, | |
ba9703b0 | 61 | layout: TyAndLayout<'tcx>, |
a1dfa0c6 | 62 | ) -> Self { |
b7449926 | 63 | assert!(layout.is_unsized(), "tried to allocate indirect place for sized values"); |
a1dfa0c6 XL |
64 | let ptr_ty = bx.cx().tcx().mk_mut_ptr(layout.ty); |
65 | let ptr_layout = bx.cx().layout_of(ptr_ty); | |
e1599b0c | 66 | Self::alloca(bx, ptr_layout) |
b7449926 XL |
67 | } |
68 | ||
dfeec247 | 69 | pub fn len<Cx: ConstMethods<'tcx, Value = V>>(&self, cx: &Cx) -> V { |
ba9703b0 | 70 | if let FieldsShape::Array { count, .. } = self.layout.fields { |
ff7c6d11 | 71 | if self.layout.is_unsized() { |
ff7c6d11 | 72 | assert_eq!(count, 0); |
b7449926 | 73 | self.llextra.unwrap() |
ff7c6d11 | 74 | } else { |
a1dfa0c6 | 75 | cx.const_usize(count) |
ff7c6d11 XL |
76 | } |
77 | } else { | |
78 | bug!("unexpected layout `{:#?}` in PlaceRef::len", self.layout) | |
79 | } | |
80 | } | |
a1dfa0c6 | 81 | } |
ff7c6d11 | 82 | |
dc9dc135 | 83 | impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> { |
ff7c6d11 | 84 | /// Access a field, at a point when the value's case is known. |
a1dfa0c6 | 85 | pub fn project_field<Bx: BuilderMethods<'a, 'tcx, Value = V>>( |
dfeec247 XL |
86 | self, |
87 | bx: &mut Bx, | |
a1dfa0c6 XL |
88 | ix: usize, |
89 | ) -> Self { | |
90 | let field = self.layout.field(bx.cx(), ix); | |
ff7c6d11 | 91 | let offset = self.layout.fields.offset(ix); |
0bf4aa26 | 92 | let effective_field_align = self.align.restrict_for_offset(offset); |
ff7c6d11 | 93 | |
a1dfa0c6 | 94 | let mut simple = || { |
1b1a35ee XL |
95 | let llval = match self.layout.abi { |
96 | _ if offset.bytes() == 0 => { | |
97 | // Unions and newtypes only use an offset of 0. | |
98 | // Also handles the first field of Scalar, ScalarPair, and Vector layouts. | |
99 | self.llval | |
100 | } | |
c295e0f8 | 101 | Abi::ScalarPair(a, b) |
04454e1e | 102 | if offset == a.size(bx.cx()).align_to(b.align(bx.cx()).abi) => |
1b1a35ee XL |
103 | { |
104 | // Offset matches second field. | |
94222f64 XL |
105 | let ty = bx.backend_type(self.layout); |
106 | bx.struct_gep(ty, self.llval, 1) | |
1b1a35ee XL |
107 | } |
108 | Abi::Scalar(_) | Abi::ScalarPair(..) | Abi::Vector { .. } if field.is_zst() => { | |
109 | // ZST fields are not included in Scalar, ScalarPair, and Vector layouts, so manually offset the pointer. | |
110 | let byte_ptr = bx.pointercast(self.llval, bx.cx().type_i8p()); | |
94222f64 | 111 | bx.gep(bx.cx().type_i8(), byte_ptr, &[bx.const_usize(offset.bytes())]) |
1b1a35ee XL |
112 | } |
113 | Abi::Scalar(_) | Abi::ScalarPair(..) => { | |
114 | // All fields of Scalar and ScalarPair layouts must have been handled by this point. | |
115 | // Vector layouts have additional fields for each element of the vector, so don't panic in that case. | |
116 | bug!( | |
117 | "offset of non-ZST field `{:?}` does not match layout `{:#?}`", | |
118 | field, | |
119 | self.layout | |
120 | ); | |
121 | } | |
94222f64 XL |
122 | _ => { |
123 | let ty = bx.backend_type(self.layout); | |
124 | bx.struct_gep(ty, self.llval, bx.cx().backend_field_index(self.layout, ix)) | |
125 | } | |
ff7c6d11 XL |
126 | }; |
127 | PlaceRef { | |
dc9dc135 | 128 | // HACK(eddyb): have to bitcast pointers until LLVM removes pointee types. |
a1dfa0c6 | 129 | llval: bx.pointercast(llval, bx.cx().type_ptr_to(bx.cx().backend_type(field))), |
dfeec247 | 130 | llextra: if bx.cx().type_has_metadata(field.ty) { self.llextra } else { None }, |
ff7c6d11 | 131 | layout: field, |
b7449926 | 132 | align: effective_field_align, |
ff7c6d11 XL |
133 | } |
134 | }; | |
135 | ||
136 | // Simple cases, which don't need DST adjustment: | |
137 | // * no metadata available - just log the case | |
dc9dc135 | 138 | // * known alignment - sized types, `[T]`, `str` or a foreign type |
ff7c6d11 | 139 | // * packed struct - there is no alignment padding |
1b1a35ee | 140 | match field.ty.kind() { |
b7449926 | 141 | _ if self.llextra.is_none() => { |
dfeec247 XL |
142 | debug!( |
143 | "unsized field `{}`, of `{:?}` has no metadata for adjustment", | |
144 | ix, self.llval | |
145 | ); | |
ff7c6d11 XL |
146 | return simple(); |
147 | } | |
148 | _ if !field.is_unsized() => return simple(), | |
b7449926 XL |
149 | ty::Slice(..) | ty::Str | ty::Foreign(..) => return simple(), |
150 | ty::Adt(def, _) => { | |
5e7ed085 | 151 | if def.repr().packed() { |
ff7c6d11 XL |
152 | // FIXME(eddyb) generalize the adjustment when we |
153 | // start supporting packing to larger alignments. | |
a1dfa0c6 | 154 | assert_eq!(self.layout.align.abi.bytes(), 1); |
ff7c6d11 XL |
155 | return simple(); |
156 | } | |
157 | } | |
158 | _ => {} | |
159 | } | |
160 | ||
161 | // We need to get the pointer manually now. | |
dc9dc135 | 162 | // We do this by casting to a `*i8`, then offsetting it by the appropriate amount. |
ff7c6d11 XL |
163 | // We do this instead of, say, simply adjusting the pointer from the result of a GEP |
164 | // because the field may have an arbitrary alignment in the LLVM representation | |
165 | // anyway. | |
166 | // | |
167 | // To demonstrate: | |
ff7c6d11 | 168 | // |
dc9dc135 XL |
169 | // struct Foo<T: ?Sized> { |
170 | // x: u16, | |
171 | // y: T | |
172 | // } | |
173 | // | |
174 | // The type `Foo<Foo<Trait>>` is represented in LLVM as `{ u16, { u16, u8 }}`, meaning that | |
ff7c6d11 XL |
175 | // the `y` field has 16-bit alignment. |
176 | ||
177 | let meta = self.llextra; | |
178 | ||
a1dfa0c6 | 179 | let unaligned_offset = bx.cx().const_usize(offset.bytes()); |
ff7c6d11 XL |
180 | |
181 | // Get the alignment of the field | |
2c00a5a8 | 182 | let (_, unsized_align) = glue::size_and_align_of_dst(bx, field.ty, meta); |
ff7c6d11 | 183 | |
5869c6ff XL |
184 | // Bump the unaligned offset up to the appropriate alignment |
185 | let offset = round_up_const_value_to_alignment(bx, unaligned_offset, unsized_align); | |
ff7c6d11 | 186 | |
b7449926 | 187 | debug!("struct_field_ptr: DST field offset: {:?}", offset); |
ff7c6d11 | 188 | |
dc9dc135 | 189 | // Cast and adjust pointer. |
a1dfa0c6 | 190 | let byte_ptr = bx.pointercast(self.llval, bx.cx().type_i8p()); |
94222f64 | 191 | let byte_ptr = bx.gep(bx.cx().type_i8(), byte_ptr, &[offset]); |
ff7c6d11 | 192 | |
dc9dc135 | 193 | // Finally, cast back to the type expected. |
a1dfa0c6 | 194 | let ll_fty = bx.cx().backend_type(field); |
ff7c6d11 XL |
195 | debug!("struct_field_ptr: Field type is {:?}", ll_fty); |
196 | ||
197 | PlaceRef { | |
a1dfa0c6 | 198 | llval: bx.pointercast(byte_ptr, bx.cx().type_ptr_to(ll_fty)), |
ff7c6d11 XL |
199 | llextra: self.llextra, |
200 | layout: field, | |
b7449926 | 201 | align: effective_field_align, |
ff7c6d11 XL |
202 | } |
203 | } | |
204 | ||
205 | /// Obtain the actual discriminant of a value. | |
064997fb | 206 | #[instrument(level = "trace", skip(bx))] |
a1dfa0c6 XL |
207 | pub fn codegen_get_discr<Bx: BuilderMethods<'a, 'tcx, Value = V>>( |
208 | self, | |
209 | bx: &mut Bx, | |
dfeec247 | 210 | cast_to: Ty<'tcx>, |
a1dfa0c6 XL |
211 | ) -> V { |
212 | let cast_to = bx.cx().immediate_backend_type(bx.cx().layout_of(cast_to)); | |
0bf4aa26 | 213 | if self.layout.abi.is_uninhabited() { |
a1dfa0c6 | 214 | return bx.cx().const_undef(cast_to); |
83c7162d | 215 | } |
f035d41b | 216 | let (tag_scalar, tag_encoding, tag_field) = match self.layout.variants { |
ba9703b0 | 217 | Variants::Single { index } => { |
dfeec247 XL |
218 | let discr_val = self |
219 | .layout | |
220 | .ty | |
221 | .discriminant_for_variant(bx.cx().tcx(), index) | |
48663c56 | 222 | .map_or(index.as_u32() as u128, |discr| discr.val); |
a1dfa0c6 | 223 | return bx.cx().const_uint_big(cast_to, discr_val); |
ff7c6d11 | 224 | } |
c295e0f8 | 225 | Variants::Multiple { tag, ref tag_encoding, tag_field, .. } => { |
f035d41b | 226 | (tag, tag_encoding, tag_field) |
532ac7d7 XL |
227 | } |
228 | }; | |
ff7c6d11 | 229 | |
416331ca | 230 | // Read the tag/niche-encoded discriminant from memory. |
f035d41b XL |
231 | let tag = self.project_field(bx, tag_field); |
232 | let tag = bx.load_operand(tag); | |
416331ca XL |
233 | |
234 | // Decode the discriminant (specifically if it's niche-encoded). | |
f035d41b XL |
235 | match *tag_encoding { |
236 | TagEncoding::Direct => { | |
04454e1e | 237 | let signed = match tag_scalar.primitive() { |
94b46f34 | 238 | // We use `i1` for bytes that are always `0` or `1`, |
0731742a | 239 | // e.g., `#[repr(i8)] enum E { A, B }`, but we can't |
94b46f34 | 240 | // let LLVM interpret the `i1` as signed, because |
dc9dc135 | 241 | // then `i1 1` (i.e., `E::B`) is effectively `i8 -1`. |
f035d41b | 242 | Int(_, signed) => !tag_scalar.is_bool() && signed, |
dfeec247 | 243 | _ => false, |
ff7c6d11 | 244 | }; |
f035d41b | 245 | bx.intcast(tag.immediate(), cast_to, signed) |
ff7c6d11 | 246 | } |
f2b60f7d | 247 | TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start } => { |
416331ca XL |
248 | // Rebase from niche values to discriminants, and check |
249 | // whether the result is in range for the niche variants. | |
f035d41b XL |
250 | let niche_llty = bx.cx().immediate_backend_type(tag.layout); |
251 | let tag = tag.immediate(); | |
416331ca XL |
252 | |
253 | // We first compute the "relative discriminant" (wrt `niche_variants`), | |
254 | // that is, if `n = niche_variants.end() - niche_variants.start()`, | |
255 | // we remap `niche_start..=niche_start + n` (which may wrap around) | |
256 | // to (non-wrap-around) `0..=n`, to be able to check whether the | |
257 | // discriminant corresponds to a niche variant with one comparison. | |
258 | // We also can't go directly to the (variant index) discriminant | |
259 | // and check that it is in the range `niche_variants`, because | |
260 | // that might not fit in the same type, on top of needing an extra | |
261 | // comparison (see also the comment on `let niche_discr`). | |
262 | let relative_discr = if niche_start == 0 { | |
263 | // Avoid subtracting `0`, which wouldn't work for pointers. | |
264 | // FIXME(eddyb) check the actual primitive type here. | |
f035d41b | 265 | tag |
416331ca | 266 | } else { |
f035d41b | 267 | bx.sub(tag, bx.cx().const_uint_big(niche_llty, niche_start)) |
416331ca XL |
268 | }; |
269 | let relative_max = niche_variants.end().as_u32() - niche_variants.start().as_u32(); | |
3dfed10e XL |
270 | let is_niche = if relative_max == 0 { |
271 | // Avoid calling `const_uint`, which wouldn't work for pointers. | |
272 | // Also use canonical == 0 instead of non-canonical u<= 0. | |
273 | // FIXME(eddyb) check the actual primitive type here. | |
274 | bx.icmp(IntPredicate::IntEQ, relative_discr, bx.cx().const_null(niche_llty)) | |
275 | } else { | |
276 | let relative_max = bx.cx().const_uint(niche_llty, relative_max as u64); | |
416331ca XL |
277 | bx.icmp(IntPredicate::IntULE, relative_discr, relative_max) |
278 | }; | |
279 | ||
280 | // NOTE(eddyb) this addition needs to be performed on the final | |
281 | // type, in case the niche itself can't represent all variant | |
282 | // indices (e.g. `u8` niche with more than `256` variants, | |
283 | // but enough uninhabited variants so that the remaining variants | |
284 | // fit in the niche). | |
285 | // In other words, `niche_variants.end - niche_variants.start` | |
286 | // is representable in the niche, but `niche_variants.end` | |
287 | // might not be, in extreme cases. | |
288 | let niche_discr = { | |
289 | let relative_discr = if relative_max == 0 { | |
290 | // HACK(eddyb) since we have only one niche, we know which | |
291 | // one it is, and we can avoid having a dynamic value here. | |
292 | bx.cx().const_uint(cast_to, 0) | |
293 | } else { | |
294 | bx.intcast(relative_discr, cast_to, false) | |
295 | }; | |
296 | bx.add( | |
297 | relative_discr, | |
a1dfa0c6 | 298 | bx.cx().const_uint(cast_to, niche_variants.start().as_u32() as u64), |
416331ca XL |
299 | ) |
300 | }; | |
301 | ||
302 | bx.select( | |
303 | is_niche, | |
304 | niche_discr, | |
f2b60f7d | 305 | bx.cx().const_uint(cast_to, untagged_variant.as_u32() as u64), |
416331ca | 306 | ) |
ff7c6d11 XL |
307 | } |
308 | } | |
309 | } | |
310 | ||
9fa01778 | 311 | /// Sets the discriminant for a new value of the given case of the given |
ff7c6d11 | 312 | /// representation. |
a1dfa0c6 XL |
313 | pub fn codegen_set_discr<Bx: BuilderMethods<'a, 'tcx, Value = V>>( |
314 | &self, | |
315 | bx: &mut Bx, | |
dfeec247 | 316 | variant_index: VariantIdx, |
a1dfa0c6 XL |
317 | ) { |
318 | if self.layout.for_variant(bx.cx(), variant_index).abi.is_uninhabited() { | |
60c5eb7d XL |
319 | // We play it safe by using a well-defined `abort`, but we could go for immediate UB |
320 | // if that turns out to be helpful. | |
321 | bx.abort(); | |
ff7c6d11 XL |
322 | return; |
323 | } | |
324 | match self.layout.variants { | |
ba9703b0 | 325 | Variants::Single { index } => { |
ff7c6d11 XL |
326 | assert_eq!(index, variant_index); |
327 | } | |
f035d41b XL |
328 | Variants::Multiple { tag_encoding: TagEncoding::Direct, tag_field, .. } => { |
329 | let ptr = self.project_field(bx, tag_field); | |
48663c56 XL |
330 | let to = |
331 | self.layout.ty.discriminant_for_variant(bx.tcx(), variant_index).unwrap().val; | |
83c7162d | 332 | bx.store( |
a1dfa0c6 | 333 | bx.cx().const_uint_big(bx.cx().backend_type(ptr.layout), to), |
83c7162d | 334 | ptr.llval, |
dfeec247 XL |
335 | ptr.align, |
336 | ); | |
ff7c6d11 | 337 | } |
ba9703b0 | 338 | Variants::Multiple { |
f035d41b | 339 | tag_encoding: |
f2b60f7d | 340 | TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start }, |
f035d41b | 341 | tag_field, |
ff7c6d11 XL |
342 | .. |
343 | } => { | |
f2b60f7d | 344 | if variant_index != untagged_variant { |
f035d41b | 345 | let niche = self.project_field(bx, tag_field); |
a1dfa0c6 XL |
346 | let niche_llty = bx.cx().immediate_backend_type(niche.layout); |
347 | let niche_value = variant_index.as_u32() - niche_variants.start().as_u32(); | |
dfeec247 | 348 | let niche_value = (niche_value as u128).wrapping_add(niche_start); |
dc9dc135 | 349 | // FIXME(eddyb): check the actual primitive type here. |
ff7c6d11 | 350 | let niche_llval = if niche_value == 0 { |
dc9dc135 | 351 | // HACK(eddyb): using `c_null` as it works on all types. |
a1dfa0c6 | 352 | bx.cx().const_null(niche_llty) |
ff7c6d11 | 353 | } else { |
a1dfa0c6 | 354 | bx.cx().const_uint_big(niche_llty, niche_value) |
ff7c6d11 | 355 | }; |
2c00a5a8 | 356 | OperandValue::Immediate(niche_llval).store(bx, niche); |
ff7c6d11 XL |
357 | } |
358 | } | |
359 | } | |
360 | } | |
361 | ||
a1dfa0c6 XL |
362 | pub fn project_index<Bx: BuilderMethods<'a, 'tcx, Value = V>>( |
363 | &self, | |
364 | bx: &mut Bx, | |
dfeec247 | 365 | llindex: V, |
a1dfa0c6 | 366 | ) -> Self { |
69743fb6 XL |
367 | // Statically compute the offset if we can, otherwise just use the element size, |
368 | // as this will yield the lowest alignment. | |
369 | let layout = self.layout.field(bx, 0); | |
e74abb32 XL |
370 | let offset = if let Some(llindex) = bx.const_to_opt_uint(llindex) { |
371 | layout.size.checked_mul(llindex, bx).unwrap_or(layout.size) | |
69743fb6 XL |
372 | } else { |
373 | layout.size | |
374 | }; | |
375 | ||
ff7c6d11 | 376 | PlaceRef { |
94222f64 XL |
377 | llval: bx.inbounds_gep( |
378 | bx.cx().backend_type(self.layout), | |
379 | self.llval, | |
380 | &[bx.cx().const_usize(0), llindex], | |
381 | ), | |
b7449926 | 382 | llextra: None, |
69743fb6 XL |
383 | layout, |
384 | align: self.align.restrict_for_offset(offset), | |
ff7c6d11 XL |
385 | } |
386 | } | |
387 | ||
a1dfa0c6 XL |
388 | pub fn project_downcast<Bx: BuilderMethods<'a, 'tcx, Value = V>>( |
389 | &self, | |
390 | bx: &mut Bx, | |
dfeec247 | 391 | variant_index: VariantIdx, |
a1dfa0c6 | 392 | ) -> Self { |
ff7c6d11 | 393 | let mut downcast = *self; |
a1dfa0c6 | 394 | downcast.layout = self.layout.for_variant(bx.cx(), variant_index); |
ff7c6d11 XL |
395 | |
396 | // Cast to the appropriate variant struct type. | |
a1dfa0c6 XL |
397 | let variant_ty = bx.cx().backend_type(downcast.layout); |
398 | downcast.llval = bx.pointercast(downcast.llval, bx.cx().type_ptr_to(variant_ty)); | |
ff7c6d11 XL |
399 | |
400 | downcast | |
401 | } | |
402 | ||
2b03887a FG |
403 | pub fn project_type<Bx: BuilderMethods<'a, 'tcx, Value = V>>( |
404 | &self, | |
405 | bx: &mut Bx, | |
406 | ty: Ty<'tcx>, | |
407 | ) -> Self { | |
408 | let mut downcast = *self; | |
409 | downcast.layout = bx.cx().layout_of(ty); | |
410 | ||
411 | // Cast to the appropriate type. | |
412 | let variant_ty = bx.cx().backend_type(downcast.layout); | |
413 | downcast.llval = bx.pointercast(downcast.llval, bx.cx().type_ptr_to(variant_ty)); | |
414 | ||
415 | downcast | |
416 | } | |
417 | ||
a1dfa0c6 | 418 | pub fn storage_live<Bx: BuilderMethods<'a, 'tcx, Value = V>>(&self, bx: &mut Bx) { |
2c00a5a8 | 419 | bx.lifetime_start(self.llval, self.layout.size); |
ff7c6d11 XL |
420 | } |
421 | ||
a1dfa0c6 | 422 | pub fn storage_dead<Bx: BuilderMethods<'a, 'tcx, Value = V>>(&self, bx: &mut Bx) { |
2c00a5a8 | 423 | bx.lifetime_end(self.llval, self.layout.size); |
ff7c6d11 XL |
424 | } |
425 | } | |
426 | ||
dc9dc135 | 427 | impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { |
064997fb | 428 | #[instrument(level = "trace", skip(self, bx))] |
a1dfa0c6 XL |
429 | pub fn codegen_place( |
430 | &mut self, | |
431 | bx: &mut Bx, | |
74b04a01 | 432 | place_ref: mir::PlaceRef<'tcx>, |
a1dfa0c6 | 433 | ) -> PlaceRef<'tcx, Bx::Value> { |
a1dfa0c6 XL |
434 | let cx = self.cx; |
435 | let tcx = self.cx.tcx(); | |
ff7c6d11 | 436 | |
5099ac24 FG |
437 | let mut base = 0; |
438 | let mut cg_base = match self.locals[place_ref.local] { | |
439 | LocalRef::Place(place) => place, | |
440 | LocalRef::UnsizedPlace(place) => bx.load_operand(place).deref(cx), | |
441 | LocalRef::Operand(..) => { | |
064997fb FG |
442 | if place_ref.has_deref() { |
443 | base = 1; | |
5e7ed085 | 444 | let cg_base = self.codegen_consume( |
5099ac24 | 445 | bx, |
064997fb | 446 | mir::PlaceRef { projection: &place_ref.projection[..0], ..place_ref }, |
5e7ed085 | 447 | ); |
923072b8 | 448 | cg_base.deref(bx.cx()) |
5099ac24 | 449 | } else { |
dfeec247 | 450 | bug!("using operand local {:?} as place", place_ref); |
8faf50e0 | 451 | } |
ff7c6d11 | 452 | } |
5099ac24 FG |
453 | }; |
454 | for elem in place_ref.projection[base..].iter() { | |
923072b8 FG |
455 | cg_base = match *elem { |
456 | mir::ProjectionElem::Deref => bx.load_operand(cg_base).deref(bx.cx()), | |
5099ac24 FG |
457 | mir::ProjectionElem::Field(ref field, _) => { |
458 | cg_base.project_field(bx, field.index()) | |
459 | } | |
2b03887a | 460 | mir::ProjectionElem::OpaqueCast(ty) => cg_base.project_type(bx, ty), |
5099ac24 FG |
461 | mir::ProjectionElem::Index(index) => { |
462 | let index = &mir::Operand::Copy(mir::Place::from(index)); | |
463 | let index = self.codegen_operand(bx, index); | |
464 | let llindex = index.immediate(); | |
465 | cg_base.project_index(bx, llindex) | |
466 | } | |
467 | mir::ProjectionElem::ConstantIndex { offset, from_end: false, min_length: _ } => { | |
468 | let lloffset = bx.cx().const_usize(offset as u64); | |
469 | cg_base.project_index(bx, lloffset) | |
470 | } | |
471 | mir::ProjectionElem::ConstantIndex { offset, from_end: true, min_length: _ } => { | |
472 | let lloffset = bx.cx().const_usize(offset as u64); | |
473 | let lllen = cg_base.len(bx.cx()); | |
474 | let llindex = bx.sub(lllen, lloffset); | |
475 | cg_base.project_index(bx, llindex) | |
476 | } | |
477 | mir::ProjectionElem::Subslice { from, to, from_end } => { | |
478 | let mut subslice = cg_base.project_index(bx, bx.cx().const_usize(from as u64)); | |
479 | let projected_ty = | |
480 | PlaceTy::from_ty(cg_base.layout.ty).projection_ty(tcx, *elem).ty; | |
481 | subslice.layout = bx.cx().layout_of(self.monomorphize(projected_ty)); | |
482 | ||
483 | if subslice.layout.is_unsized() { | |
484 | assert!(from_end, "slice subslices should be `from_end`"); | |
485 | subslice.llextra = Some(bx.sub( | |
486 | cg_base.llextra.unwrap(), | |
487 | bx.cx().const_usize((from as u64) + (to as u64)), | |
488 | )); | |
ff7c6d11 | 489 | } |
5099ac24 FG |
490 | |
491 | // Cast the place pointer type to the new | |
492 | // array or slice type (`*[%_; new_len]`). | |
493 | subslice.llval = bx.pointercast( | |
494 | subslice.llval, | |
495 | bx.cx().type_ptr_to(bx.cx().backend_type(subslice.layout)), | |
496 | ); | |
497 | ||
498 | subslice | |
ff7c6d11 | 499 | } |
5099ac24 FG |
500 | mir::ProjectionElem::Downcast(_, v) => cg_base.project_downcast(bx, v), |
501 | }; | |
502 | } | |
503 | debug!("codegen_place(place={:?}) => {:?}", place_ref, cg_base); | |
504 | cg_base | |
ff7c6d11 XL |
505 | } |
506 | ||
74b04a01 | 507 | pub fn monomorphized_place_ty(&self, place_ref: mir::PlaceRef<'tcx>) -> Ty<'tcx> { |
a1dfa0c6 | 508 | let tcx = self.cx.tcx(); |
5869c6ff | 509 | let place_ty = place_ref.ty(self.mir, tcx); |
fc512014 | 510 | self.monomorphize(place_ty.ty) |
ff7c6d11 XL |
511 | } |
512 | } | |
5869c6ff XL |
513 | |
514 | fn round_up_const_value_to_alignment<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( | |
515 | bx: &mut Bx, | |
516 | value: Bx::Value, | |
517 | align: Bx::Value, | |
518 | ) -> Bx::Value { | |
519 | // In pseudo code: | |
520 | // | |
521 | // if value & (align - 1) == 0 { | |
522 | // value | |
523 | // } else { | |
524 | // (value & !(align - 1)) + align | |
525 | // } | |
526 | // | |
527 | // Usually this is written without branches as | |
528 | // | |
529 | // (value + align - 1) & !(align - 1) | |
530 | // | |
531 | // But this formula cannot take advantage of constant `value`. E.g. if `value` is known | |
532 | // at compile time to be `1`, this expression should be optimized to `align`. However, | |
533 | // optimization only holds if `align` is a power of two. Since the optimizer doesn't know | |
534 | // that `align` is a power of two, it cannot perform this optimization. | |
535 | // | |
536 | // Instead we use | |
537 | // | |
538 | // value + (-value & (align - 1)) | |
539 | // | |
540 | // Since `align` is used only once, the expression can be optimized. For `value = 0` | |
541 | // its optimized to `0` even in debug mode. | |
542 | // | |
543 | // NB: The previous version of this code used | |
544 | // | |
545 | // (value + align - 1) & -align | |
546 | // | |
547 | // Even though `-align == !(align - 1)`, LLVM failed to optimize this even for | |
548 | // `value = 0`. Bug report: https://bugs.llvm.org/show_bug.cgi?id=48559 | |
549 | let one = bx.const_usize(1); | |
550 | let align_minus_1 = bx.sub(align, one); | |
551 | let neg_value = bx.neg(value); | |
552 | let offset = bx.and(neg_value, align_minus_1); | |
553 | bx.add(value, offset) | |
554 | } |