1 use crate::middle
::codegen_fn_attrs
::CodegenFnAttrFlags
;
2 use crate::mir
::{GeneratorLayout, GeneratorSavedLocal}
;
3 use crate::ty
::normalize_erasing_regions
::NormalizationError
;
4 use crate::ty
::subst
::Subst
;
5 use crate::ty
::{self, subst::SubstsRef, ReprOptions, Ty, TyCtxt, TypeFoldable}
;
7 use rustc_attr
as attr
;
8 use rustc_data_structures
::intern
::Interned
;
10 use rustc_hir
::lang_items
::LangItem
;
11 use rustc_index
::bit_set
::BitSet
;
12 use rustc_index
::vec
::{Idx, IndexVec}
;
13 use rustc_session
::{config::OptLevel, DataTypeKind, FieldInfo, SizeKind, VariantInfo}
;
14 use rustc_span
::symbol
::Symbol
;
15 use rustc_span
::{Span, DUMMY_SP}
;
16 use rustc_target
::abi
::call
::{
17 ArgAbi
, ArgAttribute
, ArgAttributes
, ArgExtension
, Conv
, FnAbi
, PassMode
, Reg
, RegKind
,
19 use rustc_target
::abi
::*;
20 use rustc_target
::spec
::{abi::Abi as SpecAbi, HasTargetSpec, PanicStrategy, Target}
;
25 use std
::num
::NonZeroUsize
;
28 use rand
::{seq::SliceRandom, SeedableRng}
;
29 use rand_xoshiro
::Xoshiro128StarStar
;
31 pub fn provide(providers
: &mut ty
::query
::Providers
) {
33 ty
::query
::Providers { layout_of, fn_abi_of_fn_ptr, fn_abi_of_instance, ..*providers }
;
36 pub trait IntegerExt
{
37 fn to_ty
<'tcx
>(&self, tcx
: TyCtxt
<'tcx
>, signed
: bool
) -> Ty
<'tcx
>;
38 fn from_attr
<C
: HasDataLayout
>(cx
: &C
, ity
: attr
::IntType
) -> Integer
;
39 fn from_int_ty
<C
: HasDataLayout
>(cx
: &C
, ity
: ty
::IntTy
) -> Integer
;
40 fn from_uint_ty
<C
: HasDataLayout
>(cx
: &C
, uty
: ty
::UintTy
) -> Integer
;
50 impl IntegerExt
for Integer
{
52 fn to_ty
<'tcx
>(&self, tcx
: TyCtxt
<'tcx
>, signed
: bool
) -> Ty
<'tcx
> {
53 match (*self, signed
) {
54 (I8
, false) => tcx
.types
.u8,
55 (I16
, false) => tcx
.types
.u16,
56 (I32
, false) => tcx
.types
.u32,
57 (I64
, false) => tcx
.types
.u64,
58 (I128
, false) => tcx
.types
.u128
,
59 (I8
, true) => tcx
.types
.i8,
60 (I16
, true) => tcx
.types
.i16,
61 (I32
, true) => tcx
.types
.i32,
62 (I64
, true) => tcx
.types
.i64,
63 (I128
, true) => tcx
.types
.i128
,
67 /// Gets the Integer type from an attr::IntType.
68 fn from_attr
<C
: HasDataLayout
>(cx
: &C
, ity
: attr
::IntType
) -> Integer
{
69 let dl
= cx
.data_layout();
72 attr
::SignedInt(ast
::IntTy
::I8
) | attr
::UnsignedInt(ast
::UintTy
::U8
) => I8
,
73 attr
::SignedInt(ast
::IntTy
::I16
) | attr
::UnsignedInt(ast
::UintTy
::U16
) => I16
,
74 attr
::SignedInt(ast
::IntTy
::I32
) | attr
::UnsignedInt(ast
::UintTy
::U32
) => I32
,
75 attr
::SignedInt(ast
::IntTy
::I64
) | attr
::UnsignedInt(ast
::UintTy
::U64
) => I64
,
76 attr
::SignedInt(ast
::IntTy
::I128
) | attr
::UnsignedInt(ast
::UintTy
::U128
) => I128
,
77 attr
::SignedInt(ast
::IntTy
::Isize
) | attr
::UnsignedInt(ast
::UintTy
::Usize
) => {
78 dl
.ptr_sized_integer()
83 fn from_int_ty
<C
: HasDataLayout
>(cx
: &C
, ity
: ty
::IntTy
) -> Integer
{
86 ty
::IntTy
::I16
=> I16
,
87 ty
::IntTy
::I32
=> I32
,
88 ty
::IntTy
::I64
=> I64
,
89 ty
::IntTy
::I128
=> I128
,
90 ty
::IntTy
::Isize
=> cx
.data_layout().ptr_sized_integer(),
93 fn from_uint_ty
<C
: HasDataLayout
>(cx
: &C
, ity
: ty
::UintTy
) -> Integer
{
96 ty
::UintTy
::U16
=> I16
,
97 ty
::UintTy
::U32
=> I32
,
98 ty
::UintTy
::U64
=> I64
,
99 ty
::UintTy
::U128
=> I128
,
100 ty
::UintTy
::Usize
=> cx
.data_layout().ptr_sized_integer(),
104 /// Finds the appropriate Integer type and signedness for the given
105 /// signed discriminant range and `#[repr]` attribute.
106 /// N.B.: `u128` values above `i128::MAX` will be treated as signed, but
107 /// that shouldn't affect anything, other than maybe debuginfo.
114 ) -> (Integer
, bool
) {
115 // Theoretically, negative values could be larger in unsigned representation
116 // than the unsigned representation of the signed minimum. However, if there
117 // are any negative values, the only valid unsigned representation is u128
118 // which can fit all i128 values, so the result remains unaffected.
119 let unsigned_fit
= Integer
::fit_unsigned(cmp
::max(min
as u128
, max
as u128
));
120 let signed_fit
= cmp
::max(Integer
::fit_signed(min
), Integer
::fit_signed(max
));
122 if let Some(ity
) = repr
.int
{
123 let discr
= Integer
::from_attr(&tcx
, ity
);
124 let fit
= if ity
.is_signed() { signed_fit }
else { unsigned_fit }
;
127 "Integer::repr_discr: `#[repr]` hint too small for \
128 discriminant range of enum `{}",
132 return (discr
, ity
.is_signed());
135 let at_least
= if repr
.c() {
136 // This is usually I32, however it can be different on some platforms,
137 // notably hexagon and arm-none/thumb-none
138 tcx
.data_layout().c_enum_min_size
140 // repr(Rust) enums try to be as small as possible
144 // If there are no negative values, we can use the unsigned fit.
146 (cmp
::max(unsigned_fit
, at_least
), false)
148 (cmp
::max(signed_fit
, at_least
), true)
153 pub trait PrimitiveExt
{
154 fn to_ty
<'tcx
>(&self, tcx
: TyCtxt
<'tcx
>) -> Ty
<'tcx
>;
155 fn to_int_ty
<'tcx
>(&self, tcx
: TyCtxt
<'tcx
>) -> Ty
<'tcx
>;
158 impl PrimitiveExt
for Primitive
{
160 fn to_ty
<'tcx
>(&self, tcx
: TyCtxt
<'tcx
>) -> Ty
<'tcx
> {
162 Int(i
, signed
) => i
.to_ty(tcx
, signed
),
163 F32
=> tcx
.types
.f32,
164 F64
=> tcx
.types
.f64,
165 Pointer
=> tcx
.mk_mut_ptr(tcx
.mk_unit()),
169 /// Return an *integer* type matching this primitive.
170 /// Useful in particular when dealing with enum discriminants.
172 fn to_int_ty
<'tcx
>(&self, tcx
: TyCtxt
<'tcx
>) -> Ty
<'tcx
> {
174 Int(i
, signed
) => i
.to_ty(tcx
, signed
),
175 Pointer
=> tcx
.types
.usize,
176 F32
| F64
=> bug
!("floats do not have an int type"),
181 /// The first half of a fat pointer.
183 /// - For a trait object, this is the address of the box.
184 /// - For a slice, this is the base address.
185 pub const FAT_PTR_ADDR
: usize = 0;
187 /// The second half of a fat pointer.
189 /// - For a trait object, this is the address of the vtable.
190 /// - For a slice, this is the length.
191 pub const FAT_PTR_EXTRA
: usize = 1;
193 /// The maximum supported number of lanes in a SIMD vector.
195 /// This value is selected based on backend support:
196 /// * LLVM does not appear to have a vector width limit.
197 /// * Cranelift stores the base-2 log of the lane count in a 4 bit integer.
198 pub const MAX_SIMD_LANES
: u64 = 1 << 0xF;
200 #[derive(Copy, Clone, Debug, HashStable, TyEncodable, TyDecodable)]
201 pub enum LayoutError
<'tcx
> {
203 SizeOverflow(Ty
<'tcx
>),
204 NormalizationFailure(Ty
<'tcx
>, NormalizationError
<'tcx
>),
207 impl<'tcx
> fmt
::Display
for LayoutError
<'tcx
> {
208 fn fmt(&self, f
: &mut fmt
::Formatter
<'_
>) -> fmt
::Result
{
210 LayoutError
::Unknown(ty
) => write
!(f
, "the type `{}` has an unknown layout", ty
),
211 LayoutError
::SizeOverflow(ty
) => {
212 write
!(f
, "values of the type `{}` are too big for the current architecture", ty
)
214 LayoutError
::NormalizationFailure(t
, e
) => write
!(
216 "unable to determine layout for `{}` because `{}` cannot be normalized",
218 e
.get_type_for_failure()
224 #[instrument(skip(tcx, query), level = "debug")]
227 query
: ty
::ParamEnvAnd
<'tcx
, Ty
<'tcx
>>,
228 ) -> Result
<TyAndLayout
<'tcx
>, LayoutError
<'tcx
>> {
229 ty
::tls
::with_related_context(tcx
, move |icx
| {
230 let (param_env
, ty
) = query
.into_parts();
233 if !tcx
.recursion_limit().value_within_limit(icx
.layout_depth
) {
234 tcx
.sess
.fatal(&format
!("overflow representing the type `{}`", ty
));
237 // Update the ImplicitCtxt to increase the layout_depth
238 let icx
= ty
::tls
::ImplicitCtxt { layout_depth: icx.layout_depth + 1, ..icx.clone() }
;
240 ty
::tls
::enter_context(&icx
, |_
| {
241 let param_env
= param_env
.with_reveal_all_normalized(tcx
);
242 let unnormalized_ty
= ty
;
244 // FIXME: We might want to have two different versions of `layout_of`:
245 // One that can be called after typecheck has completed and can use
246 // `normalize_erasing_regions` here and another one that can be called
247 // before typecheck has completed and uses `try_normalize_erasing_regions`.
248 let ty
= match tcx
.try_normalize_erasing_regions(param_env
, ty
) {
250 Err(normalization_error
) => {
251 return Err(LayoutError
::NormalizationFailure(ty
, normalization_error
));
255 if ty
!= unnormalized_ty
{
256 // Ensure this layout is also cached for the normalized type.
257 return tcx
.layout_of(param_env
.and(ty
));
260 let cx
= LayoutCx { tcx, param_env }
;
262 let layout
= cx
.layout_of_uncached(ty
)?
;
263 let layout
= TyAndLayout { ty, layout }
;
265 cx
.record_layout_for_printing(layout
);
267 // Type-level uninhabitedness should always imply ABI uninhabitedness.
268 if tcx
.conservative_is_privately_uninhabited(param_env
.and(ty
)) {
269 assert
!(layout
.abi
.is_uninhabited());
277 pub struct LayoutCx
<'tcx
, C
> {
279 pub param_env
: ty
::ParamEnv
<'tcx
>,
282 #[derive(Copy, Clone, Debug)]
284 /// A tuple, closure, or univariant which cannot be coerced to unsized.
286 /// A univariant, the last field of which may be coerced to unsized.
288 /// A univariant, but with a prefix of an arbitrary size & alignment (e.g., enum tag).
289 Prefixed(Size
, Align
),
292 // Invert a bijective mapping, i.e. `invert(map)[y] = x` if `map[x] = y`.
293 // This is used to go between `memory_index` (source field order to memory order)
294 // and `inverse_memory_index` (memory order to source field order).
295 // See also `FieldsShape::Arbitrary::memory_index` for more details.
296 // FIXME(eddyb) build a better abstraction for permutations, if possible.
297 fn invert_mapping(map
: &[u32]) -> Vec
<u32> {
298 let mut inverse
= vec
![0; map
.len()];
299 for i
in 0..map
.len() {
300 inverse
[map
[i
] as usize] = i
as u32;
305 impl<'tcx
> LayoutCx
<'tcx
, TyCtxt
<'tcx
>> {
306 fn scalar_pair(&self, a
: Scalar
, b
: Scalar
) -> LayoutS
<'tcx
> {
307 let dl
= self.data_layout();
308 let b_align
= b
.value
.align(dl
);
309 let align
= a
.value
.align(dl
).max(b_align
).max(dl
.aggregate_align
);
310 let b_offset
= a
.value
.size(dl
).align_to(b_align
.abi
);
311 let size
= (b_offset
+ b
.value
.size(dl
)).align_to(align
.abi
);
313 // HACK(nox): We iter on `b` and then `a` because `max_by_key`
314 // returns the last maximum.
315 let largest_niche
= Niche
::from_scalar(dl
, b_offset
, b
)
317 .chain(Niche
::from_scalar(dl
, Size
::ZERO
, a
))
318 .max_by_key(|niche
| niche
.available(dl
));
321 variants
: Variants
::Single { index: VariantIdx::new(0) }
,
322 fields
: FieldsShape
::Arbitrary
{
323 offsets
: vec
![Size
::ZERO
, b_offset
],
324 memory_index
: vec
![0, 1],
326 abi
: Abi
::ScalarPair(a
, b
),
333 fn univariant_uninterned(
336 fields
: &[TyAndLayout
<'_
>],
339 ) -> Result
<LayoutS
<'tcx
>, LayoutError
<'tcx
>> {
340 let dl
= self.data_layout();
341 let pack
= repr
.pack
;
342 if pack
.is_some() && repr
.align
.is_some() {
343 self.tcx
.sess
.delay_span_bug(DUMMY_SP
, "struct cannot be packed and aligned");
344 return Err(LayoutError
::Unknown(ty
));
347 let mut align
= if pack
.is_some() { dl.i8_align }
else { dl.aggregate_align }
;
349 let mut inverse_memory_index
: Vec
<u32> = (0..fields
.len() as u32).collect();
351 let optimize
= !repr
.inhibit_struct_field_reordering_opt();
354 if let StructKind
::MaybeUnsized
= kind { fields.len() - 1 }
else { fields.len() }
;
355 let optimizing
= &mut inverse_memory_index
[..end
];
356 let field_align
= |f
: &TyAndLayout
<'_
>| {
357 if let Some(pack
) = pack { f.align.abi.min(pack) }
else { f.align.abi }
360 // If `-Z randomize-layout` was enabled for the type definition we can shuffle
361 // the field ordering to try and catch some code making assumptions about layouts
362 // we don't guarantee
363 if repr
.can_randomize_type_layout() {
364 // `ReprOptions.layout_seed` is a deterministic seed that we can use to
365 // randomize field ordering with
366 let mut rng
= Xoshiro128StarStar
::seed_from_u64(repr
.field_shuffle_seed
);
368 // Shuffle the ordering of the fields
369 optimizing
.shuffle(&mut rng
);
371 // Otherwise we just leave things alone and actually optimize the type's fields
374 StructKind
::AlwaysSized
| StructKind
::MaybeUnsized
=> {
375 optimizing
.sort_by_key(|&x
| {
376 // Place ZSTs first to avoid "interesting offsets",
377 // especially with only one or two non-ZST fields.
378 let f
= &fields
[x
as usize];
379 (!f
.is_zst(), cmp
::Reverse(field_align(f
)))
383 StructKind
::Prefixed(..) => {
384 // Sort in ascending alignment so that the layout stays optimal
385 // regardless of the prefix
386 optimizing
.sort_by_key(|&x
| field_align(&fields
[x
as usize]));
390 // FIXME(Kixiron): We can always shuffle fields within a given alignment class
391 // regardless of the status of `-Z randomize-layout`
395 // inverse_memory_index holds field indices by increasing memory offset.
396 // That is, if field 5 has offset 0, the first element of inverse_memory_index is 5.
397 // We now write field offsets to the corresponding offset slot;
398 // field 5 with offset 0 puts 0 in offsets[5].
399 // At the bottom of this function, we invert `inverse_memory_index` to
400 // produce `memory_index` (see `invert_mapping`).
402 let mut sized
= true;
403 let mut offsets
= vec
![Size
::ZERO
; fields
.len()];
404 let mut offset
= Size
::ZERO
;
405 let mut largest_niche
= None
;
406 let mut largest_niche_available
= 0;
408 if let StructKind
::Prefixed(prefix_size
, prefix_align
) = kind
{
410 if let Some(pack
) = pack { prefix_align.min(pack) }
else { prefix_align }
;
411 align
= align
.max(AbiAndPrefAlign
::new(prefix_align
));
412 offset
= prefix_size
.align_to(prefix_align
);
415 for &i
in &inverse_memory_index
{
416 let field
= fields
[i
as usize];
418 self.tcx
.sess
.delay_span_bug(
421 "univariant: field #{} of `{}` comes after unsized field",
428 if field
.is_unsized() {
432 // Invariant: offset < dl.obj_size_bound() <= 1<<61
433 let field_align
= if let Some(pack
) = pack
{
434 field
.align
.min(AbiAndPrefAlign
::new(pack
))
438 offset
= offset
.align_to(field_align
.abi
);
439 align
= align
.max(field_align
);
441 debug
!("univariant offset: {:?} field: {:#?}", offset
, field
);
442 offsets
[i
as usize] = offset
;
444 if !repr
.hide_niche() {
445 if let Some(mut niche
) = field
.largest_niche
{
446 let available
= niche
.available(dl
);
447 if available
> largest_niche_available
{
448 largest_niche_available
= available
;
449 niche
.offset
+= offset
;
450 largest_niche
= Some(niche
);
455 offset
= offset
.checked_add(field
.size
, dl
).ok_or(LayoutError
::SizeOverflow(ty
))?
;
458 if let Some(repr_align
) = repr
.align
{
459 align
= align
.max(AbiAndPrefAlign
::new(repr_align
));
462 debug
!("univariant min_size: {:?}", offset
);
463 let min_size
= offset
;
465 // As stated above, inverse_memory_index holds field indices by increasing offset.
466 // This makes it an already-sorted view of the offsets vec.
467 // To invert it, consider:
468 // If field 5 has offset 0, offsets[0] is 5, and memory_index[5] should be 0.
469 // Field 5 would be the first element, so memory_index is i:
470 // Note: if we didn't optimize, it's already right.
473 if optimize { invert_mapping(&inverse_memory_index) }
else { inverse_memory_index }
;
475 let size
= min_size
.align_to(align
.abi
);
476 let mut abi
= Abi
::Aggregate { sized }
;
478 // Unpack newtype ABIs and find scalar pairs.
479 if sized
&& size
.bytes() > 0 {
480 // All other fields must be ZSTs.
481 let mut non_zst_fields
= fields
.iter().enumerate().filter(|&(_
, f
)| !f
.is_zst());
483 match (non_zst_fields
.next(), non_zst_fields
.next(), non_zst_fields
.next()) {
484 // We have exactly one non-ZST field.
485 (Some((i
, field
)), None
, None
) => {
486 // Field fills the struct and it has a scalar or scalar pair ABI.
487 if offsets
[i
].bytes() == 0 && align
.abi
== field
.align
.abi
&& size
== field
.size
490 // For plain scalars, or vectors of them, we can't unpack
491 // newtypes for `#[repr(C)]`, as that affects C ABIs.
492 Abi
::Scalar(_
) | Abi
::Vector { .. }
if optimize
=> {
495 // But scalar pairs are Rust-specific and get
496 // treated as aggregates by C ABIs anyway.
497 Abi
::ScalarPair(..) => {
505 // Two non-ZST fields, and they're both scalars.
510 layout
: Layout(Interned(&LayoutS { abi: Abi::Scalar(a), .. }
, _
)),
517 layout
: Layout(Interned(&LayoutS { abi: Abi::Scalar(b), .. }
, _
)),
523 // Order by the memory placement, not source order.
524 let ((i
, a
), (j
, b
)) =
525 if offsets
[i
] < offsets
[j
] { ((i, a), (j, b)) }
else { ((j, b), (i, a)) }
;
526 let pair
= self.scalar_pair(a
, b
);
527 let pair_offsets
= match pair
.fields
{
528 FieldsShape
::Arbitrary { ref offsets, ref memory_index }
=> {
529 assert_eq
!(memory_index
, &[0, 1]);
534 if offsets
[i
] == pair_offsets
[0]
535 && offsets
[j
] == pair_offsets
[1]
536 && align
== pair
.align
539 // We can use `ScalarPair` only when it matches our
540 // already computed layout (including `#[repr(C)]`).
549 if fields
.iter().any(|f
| f
.abi
.is_uninhabited()) {
550 abi
= Abi
::Uninhabited
;
554 variants
: Variants
::Single { index: VariantIdx::new(0) }
,
555 fields
: FieldsShape
::Arbitrary { offsets, memory_index }
,
563 fn layout_of_uncached(&self, ty
: Ty
<'tcx
>) -> Result
<Layout
<'tcx
>, LayoutError
<'tcx
>> {
565 let param_env
= self.param_env
;
566 let dl
= self.data_layout();
567 let scalar_unit
= |value
: Primitive
| {
568 let size
= value
.size(dl
);
569 assert
!(size
.bits() <= 128);
570 Scalar { value, valid_range: WrappingRange { start: 0, end: size.unsigned_int_max() }
}
573 |value
: Primitive
| tcx
.intern_layout(LayoutS
::scalar(self, scalar_unit(value
)));
575 let univariant
= |fields
: &[TyAndLayout
<'_
>], repr
: &ReprOptions
, kind
| {
576 Ok(tcx
.intern_layout(self.univariant_uninterned(ty
, fields
, repr
, kind
)?
))
578 debug_assert
!(!ty
.has_infer_types_or_consts());
580 Ok(match *ty
.kind() {
582 ty
::Bool
=> tcx
.intern_layout(LayoutS
::scalar(
584 Scalar { value: Int(I8, false), valid_range: WrappingRange { start: 0, end: 1 }
},
586 ty
::Char
=> tcx
.intern_layout(LayoutS
::scalar(
589 value
: Int(I32
, false),
590 valid_range
: WrappingRange { start: 0, end: 0x10FFFF }
,
593 ty
::Int(ity
) => scalar(Int(Integer
::from_int_ty(dl
, ity
), true)),
594 ty
::Uint(ity
) => scalar(Int(Integer
::from_uint_ty(dl
, ity
), false)),
595 ty
::Float(fty
) => scalar(match fty
{
596 ty
::FloatTy
::F32
=> F32
,
597 ty
::FloatTy
::F64
=> F64
,
600 let mut ptr
= scalar_unit(Pointer
);
601 ptr
.valid_range
= ptr
.valid_range
.with_start(1);
602 tcx
.intern_layout(LayoutS
::scalar(self, ptr
))
606 ty
::Never
=> tcx
.intern_layout(LayoutS
{
607 variants
: Variants
::Single { index: VariantIdx::new(0) }
,
608 fields
: FieldsShape
::Primitive
,
609 abi
: Abi
::Uninhabited
,
615 // Potentially-wide pointers.
616 ty
::Ref(_
, pointee
, _
) | ty
::RawPtr(ty
::TypeAndMut { ty: pointee, .. }
) => {
617 let mut data_ptr
= scalar_unit(Pointer
);
618 if !ty
.is_unsafe_ptr() {
619 data_ptr
.valid_range
= data_ptr
.valid_range
.with_start(1);
622 let pointee
= tcx
.normalize_erasing_regions(param_env
, pointee
);
623 if pointee
.is_sized(tcx
.at(DUMMY_SP
), param_env
) {
624 return Ok(tcx
.intern_layout(LayoutS
::scalar(self, data_ptr
)));
627 let unsized_part
= tcx
.struct_tail_erasing_lifetimes(pointee
, param_env
);
628 let metadata
= match unsized_part
.kind() {
630 return Ok(tcx
.intern_layout(LayoutS
::scalar(self, data_ptr
)));
632 ty
::Slice(_
) | ty
::Str
=> scalar_unit(Int(dl
.ptr_sized_integer(), false)),
634 let mut vtable
= scalar_unit(Pointer
);
635 vtable
.valid_range
= vtable
.valid_range
.with_start(1);
638 _
=> return Err(LayoutError
::Unknown(unsized_part
)),
641 // Effectively a (ptr, meta) tuple.
642 tcx
.intern_layout(self.scalar_pair(data_ptr
, metadata
))
645 // Arrays and slices.
646 ty
::Array(element
, mut count
) => {
647 if count
.has_projections() {
648 count
= tcx
.normalize_erasing_regions(param_env
, count
);
649 if count
.has_projections() {
650 return Err(LayoutError
::Unknown(ty
));
654 let count
= count
.try_eval_usize(tcx
, param_env
).ok_or(LayoutError
::Unknown(ty
))?
;
655 let element
= self.layout_of(element
)?
;
657 element
.size
.checked_mul(count
, dl
).ok_or(LayoutError
::SizeOverflow(ty
))?
;
660 if count
!= 0 && tcx
.conservative_is_privately_uninhabited(param_env
.and(ty
)) {
663 Abi
::Aggregate { sized: true }
666 let largest_niche
= if count
!= 0 { element.largest_niche }
else { None }
;
668 tcx
.intern_layout(LayoutS
{
669 variants
: Variants
::Single { index: VariantIdx::new(0) }
,
670 fields
: FieldsShape
::Array { stride: element.size, count }
,
673 align
: element
.align
,
677 ty
::Slice(element
) => {
678 let element
= self.layout_of(element
)?
;
679 tcx
.intern_layout(LayoutS
{
680 variants
: Variants
::Single { index: VariantIdx::new(0) }
,
681 fields
: FieldsShape
::Array { stride: element.size, count: 0 }
,
682 abi
: Abi
::Aggregate { sized: false }
,
684 align
: element
.align
,
688 ty
::Str
=> tcx
.intern_layout(LayoutS
{
689 variants
: Variants
::Single { index: VariantIdx::new(0) }
,
690 fields
: FieldsShape
::Array { stride: Size::from_bytes(1), count: 0 }
,
691 abi
: Abi
::Aggregate { sized: false }
,
698 ty
::FnDef(..) => univariant(&[], &ReprOptions
::default(), StructKind
::AlwaysSized
)?
,
699 ty
::Dynamic(..) | ty
::Foreign(..) => {
700 let mut unit
= self.univariant_uninterned(
703 &ReprOptions
::default(),
704 StructKind
::AlwaysSized
,
707 Abi
::Aggregate { ref mut sized }
=> *sized
= false,
710 tcx
.intern_layout(unit
)
713 ty
::Generator(def_id
, substs
, _
) => self.generator_layout(ty
, def_id
, substs
)?
,
715 ty
::Closure(_
, ref substs
) => {
716 let tys
= substs
.as_closure().upvar_tys();
718 &tys
.map(|ty
| self.layout_of(ty
)).collect
::<Result
<Vec
<_
>, _
>>()?
,
719 &ReprOptions
::default(),
720 StructKind
::AlwaysSized
,
726 if tys
.len() == 0 { StructKind::AlwaysSized }
else { StructKind::MaybeUnsized }
;
729 &tys
.iter().map(|k
| self.layout_of(k
)).collect
::<Result
<Vec
<_
>, _
>>()?
,
730 &ReprOptions
::default(),
735 // SIMD vector types.
736 ty
::Adt(def
, substs
) if def
.repr().simd() => {
737 if !def
.is_struct() {
738 // Should have yielded E0517 by now.
739 tcx
.sess
.delay_span_bug(
741 "#[repr(simd)] was applied to an ADT that is not a struct",
743 return Err(LayoutError
::Unknown(ty
));
746 // Supported SIMD vectors are homogeneous ADTs with at least one field:
748 // * #[repr(simd)] struct S(T, T, T, T);
749 // * #[repr(simd)] struct S { x: T, y: T, z: T, w: T }
750 // * #[repr(simd)] struct S([T; 4])
752 // where T is a primitive scalar (integer/float/pointer).
754 // SIMD vectors with zero fields are not supported.
755 // (should be caught by typeck)
756 if def
.non_enum_variant().fields
.is_empty() {
757 tcx
.sess
.fatal(&format
!("monomorphising SIMD type `{}` of zero length", ty
));
760 // Type of the first ADT field:
761 let f0_ty
= def
.non_enum_variant().fields
[0].ty(tcx
, substs
);
763 // Heterogeneous SIMD vectors are not supported:
764 // (should be caught by typeck)
765 for fi
in &def
.non_enum_variant().fields
{
766 if fi
.ty(tcx
, substs
) != f0_ty
{
767 tcx
.sess
.fatal(&format
!("monomorphising heterogeneous SIMD type `{}`", ty
));
771 // The element type and number of elements of the SIMD vector
772 // are obtained from:
774 // * the element type and length of the single array field, if
775 // the first field is of array type, or
777 // * the homogenous field type and the number of fields.
778 let (e_ty
, e_len
, is_array
) = if let ty
::Array(e_ty
, _
) = f0_ty
.kind() {
779 // First ADT field is an array:
781 // SIMD vectors with multiple array fields are not supported:
782 // (should be caught by typeck)
783 if def
.non_enum_variant().fields
.len() != 1 {
784 tcx
.sess
.fatal(&format
!(
785 "monomorphising SIMD type `{}` with more than one array field",
790 // Extract the number of elements from the layout of the array field:
792 layout
: Layout(Interned(LayoutS { fields: FieldsShape::Array { count, .. }
, .. }, _
)),
794 }) = self.layout_of(f0_ty
) else {
795 return Err(LayoutError
::Unknown(ty
));
798 (*e_ty
, *count
, true)
800 // First ADT field is not an array:
801 (f0_ty
, def
.non_enum_variant().fields
.len() as _
, false)
804 // SIMD vectors of zero length are not supported.
805 // Additionally, lengths are capped at 2^16 as a fixed maximum backends must
808 // Can't be caught in typeck if the array length is generic.
810 tcx
.sess
.fatal(&format
!("monomorphising SIMD type `{}` of zero length", ty
));
811 } else if e_len
> MAX_SIMD_LANES
{
812 tcx
.sess
.fatal(&format
!(
813 "monomorphising SIMD type `{}` of length greater than {}",
818 // Compute the ABI of the element type:
819 let e_ly
= self.layout_of(e_ty
)?
;
820 let Abi
::Scalar(e_abi
) = e_ly
.abi
else {
821 // This error isn't caught in typeck, e.g., if
822 // the element type of the vector is generic.
823 tcx
.sess
.fatal(&format
!(
824 "monomorphising SIMD type `{}` with a non-primitive-scalar \
825 (integer/float/pointer) element type `{}`",
830 // Compute the size and alignment of the vector:
831 let size
= e_ly
.size
.checked_mul(e_len
, dl
).ok_or(LayoutError
::SizeOverflow(ty
))?
;
832 let align
= dl
.vector_align(size
);
833 let size
= size
.align_to(align
.abi
);
835 // Compute the placement of the vector fields:
836 let fields
= if is_array
{
837 FieldsShape
::Arbitrary { offsets: vec![Size::ZERO], memory_index: vec![0] }
839 FieldsShape
::Array { stride: e_ly.size, count: e_len }
842 tcx
.intern_layout(LayoutS
{
843 variants
: Variants
::Single { index: VariantIdx::new(0) }
,
845 abi
: Abi
::Vector { element: e_abi, count: e_len }
,
846 largest_niche
: e_ly
.largest_niche
,
853 ty
::Adt(def
, substs
) => {
854 // Cache the field layouts.
861 .map(|field
| self.layout_of(field
.ty(tcx
, substs
)))
862 .collect
::<Result
<Vec
<_
>, _
>>()
864 .collect
::<Result
<IndexVec
<VariantIdx
, _
>, _
>>()?
;
867 if def
.repr().pack
.is_some() && def
.repr().align
.is_some() {
868 self.tcx
.sess
.delay_span_bug(
869 tcx
.def_span(def
.did()),
870 "union cannot be packed and aligned",
872 return Err(LayoutError
::Unknown(ty
));
876 if def
.repr().pack
.is_some() { dl.i8_align }
else { dl.aggregate_align }
;
878 if let Some(repr_align
) = def
.repr().align
{
879 align
= align
.max(AbiAndPrefAlign
::new(repr_align
));
882 let optimize
= !def
.repr().inhibit_union_abi_opt();
883 let mut size
= Size
::ZERO
;
884 let mut abi
= Abi
::Aggregate { sized: true }
;
885 let index
= VariantIdx
::new(0);
886 for field
in &variants
[index
] {
887 assert
!(!field
.is_unsized());
888 align
= align
.max(field
.align
);
890 // If all non-ZST fields have the same ABI, forward this ABI
891 if optimize
&& !field
.is_zst() {
892 // Normalize scalar_unit to the maximal valid range
893 let field_abi
= match field
.abi
{
894 Abi
::Scalar(x
) => Abi
::Scalar(scalar_unit(x
.value
)),
895 Abi
::ScalarPair(x
, y
) => {
896 Abi
::ScalarPair(scalar_unit(x
.value
), scalar_unit(y
.value
))
898 Abi
::Vector { element: x, count }
=> {
899 Abi
::Vector { element: scalar_unit(x.value), count }
901 Abi
::Uninhabited
| Abi
::Aggregate { .. }
=> {
902 Abi
::Aggregate { sized: true }
906 if size
== Size
::ZERO
{
907 // first non ZST: initialize 'abi'
909 } else if abi
!= field_abi
{
910 // different fields have different ABI: reset to Aggregate
911 abi
= Abi
::Aggregate { sized: true }
;
915 size
= cmp
::max(size
, field
.size
);
918 if let Some(pack
) = def
.repr().pack
{
919 align
= align
.min(AbiAndPrefAlign
::new(pack
));
922 return Ok(tcx
.intern_layout(LayoutS
{
923 variants
: Variants
::Single { index }
,
924 fields
: FieldsShape
::Union(
925 NonZeroUsize
::new(variants
[index
].len())
926 .ok_or(LayoutError
::Unknown(ty
))?
,
931 size
: size
.align_to(align
.abi
),
935 // A variant is absent if it's uninhabited and only has ZST fields.
936 // Present uninhabited variants only require space for their fields,
937 // but *not* an encoding of the discriminant (e.g., a tag value).
938 // See issue #49298 for more details on the need to leave space
939 // for non-ZST uninhabited data (mostly partial initialization).
940 let absent
= |fields
: &[TyAndLayout
<'_
>]| {
941 let uninhabited
= fields
.iter().any(|f
| f
.abi
.is_uninhabited());
942 let is_zst
= fields
.iter().all(|f
| f
.is_zst());
943 uninhabited
&& is_zst
945 let (present_first
, present_second
) = {
946 let mut present_variants
= variants
948 .filter_map(|(i
, v
)| if absent(v
) { None }
else { Some(i) }
);
949 (present_variants
.next(), present_variants
.next())
951 let present_first
= match present_first
{
952 Some(present_first
) => present_first
,
953 // Uninhabited because it has no variants, or only absent ones.
954 None
if def
.is_enum() => {
955 return Ok(tcx
.layout_of(param_env
.and(tcx
.types
.never
))?
.layout
);
957 // If it's a struct, still compute a layout so that we can still compute the
959 None
=> VariantIdx
::new(0),
962 let is_struct
= !def
.is_enum() ||
963 // Only one variant is present.
964 (present_second
.is_none() &&
965 // Representation optimizations are allowed.
966 !def
.repr().inhibit_enum_layout_opt());
968 // Struct, or univariant enum equivalent to a struct.
969 // (Typechecking will reject discriminant-sizing attrs.)
971 let v
= present_first
;
972 let kind
= if def
.is_enum() || variants
[v
].is_empty() {
973 StructKind
::AlwaysSized
975 let param_env
= tcx
.param_env(def
.did());
976 let last_field
= def
.variant(v
).fields
.last().unwrap();
978 tcx
.type_of(last_field
.did
).is_sized(tcx
.at(DUMMY_SP
), param_env
);
980 StructKind
::MaybeUnsized
982 StructKind
::AlwaysSized
986 let mut st
= self.univariant_uninterned(ty
, &variants
[v
], &def
.repr(), kind
)?
;
987 st
.variants
= Variants
::Single { index: v }
;
988 let (start
, end
) = self.tcx
.layout_scalar_valid_range(def
.did());
990 Abi
::Scalar(ref mut scalar
) | Abi
::ScalarPair(ref mut scalar
, _
) => {
991 // the asserts ensure that we are not using the
992 // `#[rustc_layout_scalar_valid_range(n)]`
993 // attribute to widen the range of anything as that would probably
994 // result in UB somewhere
995 // FIXME(eddyb) the asserts are probably not needed,
996 // as larger validity ranges would result in missed
997 // optimizations, *not* wrongly assuming the inner
998 // value is valid. e.g. unions enlarge validity ranges,
999 // because the values may be uninitialized.
1000 if let Bound
::Included(start
) = start
{
1001 // FIXME(eddyb) this might be incorrect - it doesn't
1002 // account for wrap-around (end < start) ranges.
1003 assert
!(scalar
.valid_range
.start
<= start
);
1004 scalar
.valid_range
.start
= start
;
1006 if let Bound
::Included(end
) = end
{
1007 // FIXME(eddyb) this might be incorrect - it doesn't
1008 // account for wrap-around (end < start) ranges.
1009 assert
!(scalar
.valid_range
.end
>= end
);
1010 scalar
.valid_range
.end
= end
;
1013 // Update `largest_niche` if we have introduced a larger niche.
1014 let niche
= if def
.repr().hide_niche() {
1017 Niche
::from_scalar(dl
, Size
::ZERO
, *scalar
)
1019 if let Some(niche
) = niche
{
1020 match st
.largest_niche
{
1021 Some(largest_niche
) => {
1022 // Replace the existing niche even if they're equal,
1023 // because this one is at a lower offset.
1024 if largest_niche
.available(dl
) <= niche
.available(dl
) {
1025 st
.largest_niche
= Some(niche
);
1028 None
=> st
.largest_niche
= Some(niche
),
1033 start
== Bound
::Unbounded
&& end
== Bound
::Unbounded
,
1034 "nonscalar layout for layout_scalar_valid_range type {:?}: {:#?}",
1040 return Ok(tcx
.intern_layout(st
));
1043 // At this point, we have handled all unions and
1044 // structs. (We have also handled univariant enums
1045 // that allow representation optimization.)
1046 assert
!(def
.is_enum());
1048 // The current code for niche-filling relies on variant indices
1049 // instead of actual discriminants, so dataful enums with
1050 // explicit discriminants (RFC #2363) would misbehave.
1051 let no_explicit_discriminants
= def
1054 .all(|(i
, v
)| v
.discr
== ty
::VariantDiscr
::Relative(i
.as_u32()));
1056 let mut niche_filling_layout
= None
;
1058 // Niche-filling enum optimization.
1059 if !def
.repr().inhibit_enum_layout_opt() && no_explicit_discriminants
{
1060 let mut dataful_variant
= None
;
1061 let mut niche_variants
= VariantIdx
::MAX
..=VariantIdx
::new(0);
1063 // Find one non-ZST variant.
1064 'variants
: for (v
, fields
) in variants
.iter_enumerated() {
1070 if dataful_variant
.is_none() {
1071 dataful_variant
= Some(v
);
1074 dataful_variant
= None
;
1079 niche_variants
= *niche_variants
.start().min(&v
)..=v
;
1082 if niche_variants
.start() > niche_variants
.end() {
1083 dataful_variant
= None
;
1086 if let Some(i
) = dataful_variant
{
1087 let count
= (niche_variants
.end().as_u32()
1088 - niche_variants
.start().as_u32()
1091 // Find the field with the largest niche
1092 let niche_candidate
= variants
[i
]
1095 .filter_map(|(j
, field
)| Some((j
, field
.largest_niche?
)))
1096 .max_by_key(|(_
, niche
)| niche
.available(dl
));
1098 if let Some((field_index
, niche
, (niche_start
, niche_scalar
))) =
1099 niche_candidate
.and_then(|(field_index
, niche
)| {
1100 Some((field_index
, niche
, niche
.reserve(self, count
)?
))
1103 let mut align
= dl
.aggregate_align
;
1107 let mut st
= self.univariant_uninterned(
1111 StructKind
::AlwaysSized
,
1113 st
.variants
= Variants
::Single { index: j }
;
1115 align
= align
.max(st
.align
);
1117 Ok(tcx
.intern_layout(st
))
1119 .collect
::<Result
<IndexVec
<VariantIdx
, _
>, _
>>()?
;
1121 let offset
= st
[i
].fields().offset(field_index
) + niche
.offset
;
1122 let size
= st
[i
].size();
1124 let abi
= if st
.iter().all(|v
| v
.abi().is_uninhabited()) {
1128 Abi
::Scalar(_
) => Abi
::Scalar(niche_scalar
),
1129 Abi
::ScalarPair(first
, second
) => {
1130 // We need to use scalar_unit to reset the
1131 // valid range to the maximal one for that
1132 // primitive, because only the niche is
1133 // guaranteed to be initialised, not the
1135 if offset
.bytes() == 0 {
1136 Abi
::ScalarPair(niche_scalar
, scalar_unit(second
.value
))
1138 Abi
::ScalarPair(scalar_unit(first
.value
), niche_scalar
)
1141 _
=> Abi
::Aggregate { sized: true }
,
1145 let largest_niche
= Niche
::from_scalar(dl
, offset
, niche_scalar
);
1147 niche_filling_layout
= Some(LayoutS
{
1148 variants
: Variants
::Multiple
{
1150 tag_encoding
: TagEncoding
::Niche
{
1158 fields
: FieldsShape
::Arbitrary
{
1159 offsets
: vec
![offset
],
1160 memory_index
: vec
![0],
1171 let (mut min
, mut max
) = (i128
::MAX
, i128
::MIN
);
1172 let discr_type
= def
.repr().discr_type();
1173 let bits
= Integer
::from_attr(self, discr_type
).size().bits();
1174 for (i
, discr
) in def
.discriminants(tcx
) {
1175 if variants
[i
].iter().any(|f
| f
.abi
.is_uninhabited()) {
1178 let mut x
= discr
.val
as i128
;
1179 if discr_type
.is_signed() {
1180 // sign extend the raw representation to be an i128
1181 x
= (x
<< (128 - bits
)) >> (128 - bits
);
1190 // We might have no inhabited variants, so pretend there's at least one.
1191 if (min
, max
) == (i128
::MAX
, i128
::MIN
) {
1195 assert
!(min
<= max
, "discriminant range is {}...{}", min
, max
);
1196 let (min_ity
, signed
) = Integer
::repr_discr(tcx
, ty
, &def
.repr(), min
, max
);
1198 let mut align
= dl
.aggregate_align
;
1199 let mut size
= Size
::ZERO
;
1201 // We're interested in the smallest alignment, so start large.
1202 let mut start_align
= Align
::from_bytes(256).unwrap();
1203 assert_eq
!(Integer
::for_align(dl
, start_align
), None
);
1205 // repr(C) on an enum tells us to make a (tag, union) layout,
1206 // so we need to grow the prefix alignment to be at least
1207 // the alignment of the union. (This value is used both for
1208 // determining the alignment of the overall enum, and the
1209 // determining the alignment of the payload after the tag.)
1210 let mut prefix_align
= min_ity
.align(dl
).abi
;
1212 for fields
in &variants
{
1213 for field
in fields
{
1214 prefix_align
= prefix_align
.max(field
.align
.abi
);
1219 // Create the set of structs that represent each variant.
1220 let mut layout_variants
= variants
1222 .map(|(i
, field_layouts
)| {
1223 let mut st
= self.univariant_uninterned(
1227 StructKind
::Prefixed(min_ity
.size(), prefix_align
),
1229 st
.variants
= Variants
::Single { index: i }
;
1230 // Find the first field we can't move later
1231 // to make room for a larger discriminant.
1233 st
.fields
.index_by_increasing_offset().map(|j
| field_layouts
[j
])
1235 if !field
.is_zst() || field
.align
.abi
.bytes() != 1 {
1236 start_align
= start_align
.min(field
.align
.abi
);
1240 size
= cmp
::max(size
, st
.size
);
1241 align
= align
.max(st
.align
);
1244 .collect
::<Result
<IndexVec
<VariantIdx
, _
>, _
>>()?
;
1246 // Align the maximum variant size to the largest alignment.
1247 size
= size
.align_to(align
.abi
);
1249 if size
.bytes() >= dl
.obj_size_bound() {
1250 return Err(LayoutError
::SizeOverflow(ty
));
1253 let typeck_ity
= Integer
::from_attr(dl
, def
.repr().discr_type());
1254 if typeck_ity
< min_ity
{
1255 // It is a bug if Layout decided on a greater discriminant size than typeck for
1256 // some reason at this point (based on values discriminant can take on). Mostly
1257 // because this discriminant will be loaded, and then stored into variable of
1258 // type calculated by typeck. Consider such case (a bug): typeck decided on
1259 // byte-sized discriminant, but layout thinks we need a 16-bit to store all
1260 // discriminant values. That would be a bug, because then, in codegen, in order
1261 // to store this 16-bit discriminant into 8-bit sized temporary some of the
1262 // space necessary to represent would have to be discarded (or layout is wrong
1263 // on thinking it needs 16 bits)
1265 "layout decided on a larger discriminant type ({:?}) than typeck ({:?})",
1269 // However, it is fine to make discr type however large (as an optimisation)
1270 // after this point – we’ll just truncate the value we load in codegen.
1273 // Check to see if we should use a different type for the
1274 // discriminant. We can safely use a type with the same size
1275 // as the alignment of the first field of each variant.
1276 // We increase the size of the discriminant to avoid LLVM copying
1277 // padding when it doesn't need to. This normally causes unaligned
1278 // load/stores and excessive memcpy/memset operations. By using a
1279 // bigger integer size, LLVM can be sure about its contents and
1280 // won't be so conservative.
1282 // Use the initial field alignment
1283 let mut ity
= if def
.repr().c() || def
.repr().int
.is_some() {
1286 Integer
::for_align(dl
, start_align
).unwrap_or(min_ity
)
1289 // If the alignment is not larger than the chosen discriminant size,
1290 // don't use the alignment as the final size.
1294 // Patch up the variants' first few fields.
1295 let old_ity_size
= min_ity
.size();
1296 let new_ity_size
= ity
.size();
1297 for variant
in &mut layout_variants
{
1298 match variant
.fields
{
1299 FieldsShape
::Arbitrary { ref mut offsets, .. }
=> {
1301 if *i
<= old_ity_size
{
1302 assert_eq
!(*i
, old_ity_size
);
1306 // We might be making the struct larger.
1307 if variant
.size
<= old_ity_size
{
1308 variant
.size
= new_ity_size
;
1316 let tag_mask
= ity
.size().unsigned_int_max();
1318 value
: Int(ity
, signed
),
1319 valid_range
: WrappingRange
{
1320 start
: (min
as u128
& tag_mask
),
1321 end
: (max
as u128
& tag_mask
),
1324 let mut abi
= Abi
::Aggregate { sized: true }
;
1326 // Without latter check aligned enums with custom discriminant values
1327 // Would result in ICE see the issue #92464 for more info
1328 if tag
.value
.size(dl
) == size
|| variants
.iter().all(|layout
| layout
.is_empty()) {
1329 abi
= Abi
::Scalar(tag
);
1331 // Try to use a ScalarPair for all tagged enums.
1332 let mut common_prim
= None
;
1333 for (field_layouts
, layout_variant
) in iter
::zip(&variants
, &layout_variants
) {
1334 let FieldsShape
::Arbitrary { ref offsets, .. }
= layout_variant
.fields
else {
1338 iter
::zip(field_layouts
, offsets
).filter(|p
| !p
.0.is_zst
());
1339 let (field
, offset
) = match (fields
.next(), fields
.next()) {
1340 (None
, None
) => continue,
1341 (Some(pair
), None
) => pair
,
1347 let prim
= match field
.abi
{
1348 Abi
::Scalar(scalar
) => scalar
.value
,
1354 if let Some(pair
) = common_prim
{
1355 // This is pretty conservative. We could go fancier
1356 // by conflating things like i32 and u32, or even
1357 // realising that (u8, u8) could just cohabit with
1359 if pair
!= (prim
, offset
) {
1364 common_prim
= Some((prim
, offset
));
1367 if let Some((prim
, offset
)) = common_prim
{
1368 let pair
= self.scalar_pair(tag
, scalar_unit(prim
));
1369 let pair_offsets
= match pair
.fields
{
1370 FieldsShape
::Arbitrary { ref offsets, ref memory_index }
=> {
1371 assert_eq
!(memory_index
, &[0, 1]);
1376 if pair_offsets
[0] == Size
::ZERO
1377 && pair_offsets
[1] == *offset
1378 && align
== pair
.align
1379 && size
== pair
.size
1381 // We can use `ScalarPair` only when it matches our
1382 // already computed layout (including `#[repr(C)]`).
1388 if layout_variants
.iter().all(|v
| v
.abi
.is_uninhabited()) {
1389 abi
= Abi
::Uninhabited
;
1392 let largest_niche
= Niche
::from_scalar(dl
, Size
::ZERO
, tag
);
1394 let layout_variants
=
1395 layout_variants
.into_iter().map(|v
| tcx
.intern_layout(v
)).collect();
1397 let tagged_layout
= LayoutS
{
1398 variants
: Variants
::Multiple
{
1400 tag_encoding
: TagEncoding
::Direct
,
1402 variants
: layout_variants
,
1404 fields
: FieldsShape
::Arbitrary
{
1405 offsets
: vec
![Size
::ZERO
],
1406 memory_index
: vec
![0],
1414 let best_layout
= match (tagged_layout
, niche_filling_layout
) {
1415 (tagged_layout
, Some(niche_filling_layout
)) => {
1416 // Pick the smaller layout; otherwise,
1417 // pick the layout with the larger niche; otherwise,
1418 // pick tagged as it has simpler codegen.
1419 cmp
::min_by_key(tagged_layout
, niche_filling_layout
, |layout
| {
1420 let niche_size
= layout
.largest_niche
.map_or(0, |n
| n
.available(dl
));
1421 (layout
.size
, cmp
::Reverse(niche_size
))
1424 (tagged_layout
, None
) => tagged_layout
,
1427 tcx
.intern_layout(best_layout
)
1430 // Types with no meaningful known layout.
1431 ty
::Projection(_
) | ty
::Opaque(..) => {
1432 // NOTE(eddyb) `layout_of` query should've normalized these away,
1433 // if that was possible, so there's no reason to try again here.
1434 return Err(LayoutError
::Unknown(ty
));
1437 ty
::Placeholder(..) | ty
::GeneratorWitness(..) | ty
::Infer(_
) => {
1438 bug
!("Layout::compute: unexpected type `{}`", ty
)
1441 ty
::Bound(..) | ty
::Param(_
) | ty
::Error(_
) => {
1442 return Err(LayoutError
::Unknown(ty
));
1448 /// Overlap eligibility and variant assignment for each GeneratorSavedLocal.
1449 #[derive(Clone, Debug, PartialEq)]
1450 enum SavedLocalEligibility
{
1452 Assigned(VariantIdx
),
1453 // FIXME: Use newtype_index so we aren't wasting bytes
1454 Ineligible(Option
<u32>),
1457 // When laying out generators, we divide our saved local fields into two
1458 // categories: overlap-eligible and overlap-ineligible.
1460 // Those fields which are ineligible for overlap go in a "prefix" at the
1461 // beginning of the layout, and always have space reserved for them.
1463 // Overlap-eligible fields are only assigned to one variant, so we lay
1464 // those fields out for each variant and put them right after the
1467 // Finally, in the layout details, we point to the fields from the
1468 // variants they are assigned to. It is possible for some fields to be
1469 // included in multiple variants. No field ever "moves around" in the
1470 // layout; its offset is always the same.
1472 // Also included in the layout are the upvars and the discriminant.
1473 // These are included as fields on the "outer" layout; they are not part
1475 impl<'tcx
> LayoutCx
<'tcx
, TyCtxt
<'tcx
>> {
1476 /// Compute the eligibility and assignment of each local.
1477 fn generator_saved_local_eligibility(
1479 info
: &GeneratorLayout
<'tcx
>,
1480 ) -> (BitSet
<GeneratorSavedLocal
>, IndexVec
<GeneratorSavedLocal
, SavedLocalEligibility
>) {
1481 use SavedLocalEligibility
::*;
1483 let mut assignments
: IndexVec
<GeneratorSavedLocal
, SavedLocalEligibility
> =
1484 IndexVec
::from_elem_n(Unassigned
, info
.field_tys
.len());
1486 // The saved locals not eligible for overlap. These will get
1487 // "promoted" to the prefix of our generator.
1488 let mut ineligible_locals
= BitSet
::new_empty(info
.field_tys
.len());
1490 // Figure out which of our saved locals are fields in only
1491 // one variant. The rest are deemed ineligible for overlap.
1492 for (variant_index
, fields
) in info
.variant_fields
.iter_enumerated() {
1493 for local
in fields
{
1494 match assignments
[*local
] {
1496 assignments
[*local
] = Assigned(variant_index
);
1499 // We've already seen this local at another suspension
1500 // point, so it is no longer a candidate.
1502 "removing local {:?} in >1 variant ({:?}, {:?})",
1507 ineligible_locals
.insert(*local
);
1508 assignments
[*local
] = Ineligible(None
);
1515 // Next, check every pair of eligible locals to see if they
1517 for local_a
in info
.storage_conflicts
.rows() {
1518 let conflicts_a
= info
.storage_conflicts
.count(local_a
);
1519 if ineligible_locals
.contains(local_a
) {
1523 for local_b
in info
.storage_conflicts
.iter(local_a
) {
1524 // local_a and local_b are storage live at the same time, therefore they
1525 // cannot overlap in the generator layout. The only way to guarantee
1526 // this is if they are in the same variant, or one is ineligible
1527 // (which means it is stored in every variant).
1528 if ineligible_locals
.contains(local_b
)
1529 || assignments
[local_a
] == assignments
[local_b
]
1534 // If they conflict, we will choose one to make ineligible.
1535 // This is not always optimal; it's just a greedy heuristic that
1536 // seems to produce good results most of the time.
1537 let conflicts_b
= info
.storage_conflicts
.count(local_b
);
1538 let (remove
, other
) =
1539 if conflicts_a
> conflicts_b { (local_a, local_b) }
else { (local_b, local_a) }
;
1540 ineligible_locals
.insert(remove
);
1541 assignments
[remove
] = Ineligible(None
);
1542 trace
!("removing local {:?} due to conflict with {:?}", remove
, other
);
1546 // Count the number of variants in use. If only one of them, then it is
1547 // impossible to overlap any locals in our layout. In this case it's
1548 // always better to make the remaining locals ineligible, so we can
1549 // lay them out with the other locals in the prefix and eliminate
1550 // unnecessary padding bytes.
1552 let mut used_variants
= BitSet
::new_empty(info
.variant_fields
.len());
1553 for assignment
in &assignments
{
1554 if let Assigned(idx
) = assignment
{
1555 used_variants
.insert(*idx
);
1558 if used_variants
.count() < 2 {
1559 for assignment
in assignments
.iter_mut() {
1560 *assignment
= Ineligible(None
);
1562 ineligible_locals
.insert_all();
1566 // Write down the order of our locals that will be promoted to the prefix.
1568 for (idx
, local
) in ineligible_locals
.iter().enumerate() {
1569 assignments
[local
] = Ineligible(Some(idx
as u32));
1572 debug
!("generator saved local assignments: {:?}", assignments
);
1574 (ineligible_locals
, assignments
)
1577 /// Compute the full generator layout.
1578 fn generator_layout(
1581 def_id
: hir
::def_id
::DefId
,
1582 substs
: SubstsRef
<'tcx
>,
1583 ) -> Result
<Layout
<'tcx
>, LayoutError
<'tcx
>> {
1584 use SavedLocalEligibility
::*;
1586 let subst_field
= |ty
: Ty
<'tcx
>| ty
.subst(tcx
, substs
);
1588 let Some(info
) = tcx
.generator_layout(def_id
) else {
1589 return Err(LayoutError
::Unknown(ty
));
1591 let (ineligible_locals
, assignments
) = self.generator_saved_local_eligibility(&info
);
1593 // Build a prefix layout, including "promoting" all ineligible
1594 // locals as part of the prefix. We compute the layout of all of
1595 // these fields at once to get optimal packing.
1596 let tag_index
= substs
.as_generator().prefix_tys().count();
1598 // `info.variant_fields` already accounts for the reserved variants, so no need to add them.
1599 let max_discr
= (info
.variant_fields
.len() - 1) as u128
;
1600 let discr_int
= Integer
::fit_unsigned(max_discr
);
1601 let discr_int_ty
= discr_int
.to_ty(tcx
, false);
1603 value
: Primitive
::Int(discr_int
, false),
1604 valid_range
: WrappingRange { start: 0, end: max_discr }
,
1606 let tag_layout
= self.tcx
.intern_layout(LayoutS
::scalar(self, tag
));
1607 let tag_layout
= TyAndLayout { ty: discr_int_ty, layout: tag_layout }
;
1609 let promoted_layouts
= ineligible_locals
1611 .map(|local
| subst_field(info
.field_tys
[local
]))
1612 .map(|ty
| tcx
.mk_maybe_uninit(ty
))
1613 .map(|ty
| self.layout_of(ty
));
1614 let prefix_layouts
= substs
1617 .map(|ty
| self.layout_of(ty
))
1618 .chain(iter
::once(Ok(tag_layout
)))
1619 .chain(promoted_layouts
)
1620 .collect
::<Result
<Vec
<_
>, _
>>()?
;
1621 let prefix
= self.univariant_uninterned(
1624 &ReprOptions
::default(),
1625 StructKind
::AlwaysSized
,
1628 let (prefix_size
, prefix_align
) = (prefix
.size
, prefix
.align
);
1630 // Split the prefix layout into the "outer" fields (upvars and
1631 // discriminant) and the "promoted" fields. Promoted fields will
1632 // get included in each variant that requested them in
1634 debug
!("prefix = {:#?}", prefix
);
1635 let (outer_fields
, promoted_offsets
, promoted_memory_index
) = match prefix
.fields
{
1636 FieldsShape
::Arbitrary { mut offsets, memory_index }
=> {
1637 let mut inverse_memory_index
= invert_mapping(&memory_index
);
1639 // "a" (`0..b_start`) and "b" (`b_start..`) correspond to
1640 // "outer" and "promoted" fields respectively.
1641 let b_start
= (tag_index
+ 1) as u32;
1642 let offsets_b
= offsets
.split_off(b_start
as usize);
1643 let offsets_a
= offsets
;
1645 // Disentangle the "a" and "b" components of `inverse_memory_index`
1646 // by preserving the order but keeping only one disjoint "half" each.
1647 // FIXME(eddyb) build a better abstraction for permutations, if possible.
1648 let inverse_memory_index_b
: Vec
<_
> =
1649 inverse_memory_index
.iter().filter_map(|&i
| i
.checked_sub(b_start
)).collect();
1650 inverse_memory_index
.retain(|&i
| i
< b_start
);
1651 let inverse_memory_index_a
= inverse_memory_index
;
1653 // Since `inverse_memory_index_{a,b}` each only refer to their
1654 // respective fields, they can be safely inverted
1655 let memory_index_a
= invert_mapping(&inverse_memory_index_a
);
1656 let memory_index_b
= invert_mapping(&inverse_memory_index_b
);
1659 FieldsShape
::Arbitrary { offsets: offsets_a, memory_index: memory_index_a }
;
1660 (outer_fields
, offsets_b
, memory_index_b
)
1665 let mut size
= prefix
.size
;
1666 let mut align
= prefix
.align
;
1670 .map(|(index
, variant_fields
)| {
1671 // Only include overlap-eligible fields when we compute our variant layout.
1672 let variant_only_tys
= variant_fields
1674 .filter(|local
| match assignments
[**local
] {
1675 Unassigned
=> bug
!(),
1676 Assigned(v
) if v
== index
=> true,
1677 Assigned(_
) => bug
!("assignment does not match variant"),
1678 Ineligible(_
) => false,
1680 .map(|local
| subst_field(info
.field_tys
[*local
]));
1682 let mut variant
= self.univariant_uninterned(
1685 .map(|ty
| self.layout_of(ty
))
1686 .collect
::<Result
<Vec
<_
>, _
>>()?
,
1687 &ReprOptions
::default(),
1688 StructKind
::Prefixed(prefix_size
, prefix_align
.abi
),
1690 variant
.variants
= Variants
::Single { index }
;
1692 let FieldsShape
::Arbitrary { offsets, memory_index }
= variant
.fields
else {
1696 // Now, stitch the promoted and variant-only fields back together in
1697 // the order they are mentioned by our GeneratorLayout.
1698 // Because we only use some subset (that can differ between variants)
1699 // of the promoted fields, we can't just pick those elements of the
1700 // `promoted_memory_index` (as we'd end up with gaps).
1701 // So instead, we build an "inverse memory_index", as if all of the
1702 // promoted fields were being used, but leave the elements not in the
1703 // subset as `INVALID_FIELD_IDX`, which we can filter out later to
1704 // obtain a valid (bijective) mapping.
1705 const INVALID_FIELD_IDX
: u32 = !0;
1706 let mut combined_inverse_memory_index
=
1707 vec
![INVALID_FIELD_IDX
; promoted_memory_index
.len() + memory_index
.len()];
1708 let mut offsets_and_memory_index
= iter
::zip(offsets
, memory_index
);
1709 let combined_offsets
= variant_fields
1713 let (offset
, memory_index
) = match assignments
[*local
] {
1714 Unassigned
=> bug
!(),
1716 let (offset
, memory_index
) =
1717 offsets_and_memory_index
.next().unwrap();
1718 (offset
, promoted_memory_index
.len() as u32 + memory_index
)
1720 Ineligible(field_idx
) => {
1721 let field_idx
= field_idx
.unwrap() as usize;
1722 (promoted_offsets
[field_idx
], promoted_memory_index
[field_idx
])
1725 combined_inverse_memory_index
[memory_index
as usize] = i
as u32;
1730 // Remove the unused slots and invert the mapping to obtain the
1731 // combined `memory_index` (also see previous comment).
1732 combined_inverse_memory_index
.retain(|&i
| i
!= INVALID_FIELD_IDX
);
1733 let combined_memory_index
= invert_mapping(&combined_inverse_memory_index
);
1735 variant
.fields
= FieldsShape
::Arbitrary
{
1736 offsets
: combined_offsets
,
1737 memory_index
: combined_memory_index
,
1740 size
= size
.max(variant
.size
);
1741 align
= align
.max(variant
.align
);
1742 Ok(tcx
.intern_layout(variant
))
1744 .collect
::<Result
<IndexVec
<VariantIdx
, _
>, _
>>()?
;
1746 size
= size
.align_to(align
.abi
);
1749 if prefix
.abi
.is_uninhabited() || variants
.iter().all(|v
| v
.abi().is_uninhabited()) {
1752 Abi
::Aggregate { sized: true }
1755 let layout
= tcx
.intern_layout(LayoutS
{
1756 variants
: Variants
::Multiple
{
1758 tag_encoding
: TagEncoding
::Direct
,
1759 tag_field
: tag_index
,
1762 fields
: outer_fields
,
1764 largest_niche
: prefix
.largest_niche
,
1768 debug
!("generator layout ({:?}): {:#?}", ty
, layout
);
1772 /// This is invoked by the `layout_of` query to record the final
1773 /// layout of each type.
1775 fn record_layout_for_printing(&self, layout
: TyAndLayout
<'tcx
>) {
1776 // If we are running with `-Zprint-type-sizes`, maybe record layouts
1777 // for dumping later.
1778 if self.tcx
.sess
.opts
.debugging_opts
.print_type_sizes
{
1779 self.record_layout_for_printing_outlined(layout
)
1783 fn record_layout_for_printing_outlined(&self, layout
: TyAndLayout
<'tcx
>) {
1784 // Ignore layouts that are done with non-empty environments or
1785 // non-monomorphic layouts, as the user only wants to see the stuff
1786 // resulting from the final codegen session.
1787 if layout
.ty
.has_param_types_or_consts() || !self.param_env
.caller_bounds().is_empty() {
1791 // (delay format until we actually need it)
1792 let record
= |kind
, packed
, opt_discr_size
, variants
| {
1793 let type_desc
= format
!("{:?}", layout
.ty
);
1794 self.tcx
.sess
.code_stats
.record_type_size(
1805 let adt_def
= match *layout
.ty
.kind() {
1806 ty
::Adt(ref adt_def
, _
) => {
1807 debug
!("print-type-size t: `{:?}` process adt", layout
.ty
);
1811 ty
::Closure(..) => {
1812 debug
!("print-type-size t: `{:?}` record closure", layout
.ty
);
1813 record(DataTypeKind
::Closure
, false, None
, vec
![]);
1818 debug
!("print-type-size t: `{:?}` skip non-nominal", layout
.ty
);
1823 let adt_kind
= adt_def
.adt_kind();
1824 let adt_packed
= adt_def
.repr().pack
.is_some();
1826 let build_variant_info
= |n
: Option
<Symbol
>, flds
: &[Symbol
], layout
: TyAndLayout
<'tcx
>| {
1827 let mut min_size
= Size
::ZERO
;
1828 let field_info
: Vec
<_
> = flds
1832 let field_layout
= layout
.field(self, i
);
1833 let offset
= layout
.fields
.offset(i
);
1834 let field_end
= offset
+ field_layout
.size
;
1835 if min_size
< field_end
{
1836 min_size
= field_end
;
1839 name
: name
.to_string(),
1840 offset
: offset
.bytes(),
1841 size
: field_layout
.size
.bytes(),
1842 align
: field_layout
.align
.abi
.bytes(),
1848 name
: n
.map(|n
| n
.to_string()),
1849 kind
: if layout
.is_unsized() { SizeKind::Min }
else { SizeKind::Exact }
,
1850 align
: layout
.align
.abi
.bytes(),
1851 size
: if min_size
.bytes() == 0 { layout.size.bytes() }
else { min_size.bytes() }
,
1856 match layout
.variants
{
1857 Variants
::Single { index }
=> {
1858 if !adt_def
.variants().is_empty() && layout
.fields
!= FieldsShape
::Primitive
{
1860 "print-type-size `{:#?}` variant {}",
1862 adt_def
.variant(index
).name
1864 let variant_def
= &adt_def
.variant(index
);
1865 let fields
: Vec
<_
> = variant_def
.fields
.iter().map(|f
| f
.name
).collect();
1870 vec
![build_variant_info(Some(variant_def
.name
), &fields
, layout
)],
1873 // (This case arises for *empty* enums; so give it
1875 record(adt_kind
.into(), adt_packed
, None
, vec
![]);
1879 Variants
::Multiple { tag, ref tag_encoding, .. }
=> {
1881 "print-type-size `{:#?}` adt general variants def {}",
1883 adt_def
.variants().len()
1885 let variant_infos
: Vec
<_
> = adt_def
1888 .map(|(i
, variant_def
)| {
1889 let fields
: Vec
<_
> = variant_def
.fields
.iter().map(|f
| f
.name
).collect();
1891 Some(variant_def
.name
),
1893 layout
.for_variant(self, i
),
1900 match tag_encoding
{
1901 TagEncoding
::Direct
=> Some(tag
.value
.size(self)),
1911 /// Type size "skeleton", i.e., the only information determining a type's size.
1912 /// While this is conservative, (aside from constant sizes, only pointers,
1913 /// newtypes thereof and null pointer optimized enums are allowed), it is
1914 /// enough to statically check common use cases of transmute.
1915 #[derive(Copy, Clone, Debug)]
1916 pub enum SizeSkeleton
<'tcx
> {
1917 /// Any statically computable Layout.
1920 /// A potentially-fat pointer.
1922 /// If true, this pointer is never null.
1924 /// The type which determines the unsized metadata, if any,
1925 /// of this pointer. Either a type parameter or a projection
1926 /// depending on one, with regions erased.
1931 impl<'tcx
> SizeSkeleton
<'tcx
> {
1935 param_env
: ty
::ParamEnv
<'tcx
>,
1936 ) -> Result
<SizeSkeleton
<'tcx
>, LayoutError
<'tcx
>> {
1937 debug_assert
!(!ty
.has_infer_types_or_consts());
1939 // First try computing a static layout.
1940 let err
= match tcx
.layout_of(param_env
.and(ty
)) {
1942 return Ok(SizeSkeleton
::Known(layout
.size
));
1948 ty
::Ref(_
, pointee
, _
) | ty
::RawPtr(ty
::TypeAndMut { ty: pointee, .. }
) => {
1949 let non_zero
= !ty
.is_unsafe_ptr();
1950 let tail
= tcx
.struct_tail_erasing_lifetimes(pointee
, param_env
);
1952 ty
::Param(_
) | ty
::Projection(_
) => {
1953 debug_assert
!(tail
.has_param_types_or_consts());
1954 Ok(SizeSkeleton
::Pointer { non_zero, tail: tcx.erase_regions(tail) }
)
1957 "SizeSkeleton::compute({}): layout errored ({}), yet \
1958 tail `{}` is not a type parameter or a projection",
1966 ty
::Adt(def
, substs
) => {
1967 // Only newtypes and enums w/ nullable pointer optimization.
1968 if def
.is_union() || def
.variants().is_empty() || def
.variants().len() > 2 {
1972 // Get a zero-sized variant or a pointer newtype.
1973 let zero_or_ptr_variant
= |i
| {
1974 let i
= VariantIdx
::new(i
);
1976 def
.variant(i
).fields
.iter().map(|field
| {
1977 SizeSkeleton
::compute(field
.ty(tcx
, substs
), tcx
, param_env
)
1980 for field
in fields
{
1983 SizeSkeleton
::Known(size
) => {
1984 if size
.bytes() > 0 {
1988 SizeSkeleton
::Pointer { .. }
=> {
1999 let v0
= zero_or_ptr_variant(0)?
;
2001 if def
.variants().len() == 1 {
2002 if let Some(SizeSkeleton
::Pointer { non_zero, tail }
) = v0
{
2003 return Ok(SizeSkeleton
::Pointer
{
2005 || match tcx
.layout_scalar_valid_range(def
.did()) {
2006 (Bound
::Included(start
), Bound
::Unbounded
) => start
> 0,
2007 (Bound
::Included(start
), Bound
::Included(end
)) => {
2008 0 < start
&& start
< end
2019 let v1
= zero_or_ptr_variant(1)?
;
2020 // Nullable pointer enum optimization.
2022 (Some(SizeSkeleton
::Pointer { non_zero: true, tail }
), None
)
2023 | (None
, Some(SizeSkeleton
::Pointer { non_zero: true, tail }
)) => {
2024 Ok(SizeSkeleton
::Pointer { non_zero: false, tail }
)
2030 ty
::Projection(_
) | ty
::Opaque(..) => {
2031 let normalized
= tcx
.normalize_erasing_regions(param_env
, ty
);
2032 if ty
== normalized
{
2035 SizeSkeleton
::compute(normalized
, tcx
, param_env
)
2043 pub fn same_size(self, other
: SizeSkeleton
<'_
>) -> bool
{
2044 match (self, other
) {
2045 (SizeSkeleton
::Known(a
), SizeSkeleton
::Known(b
)) => a
== b
,
2046 (SizeSkeleton
::Pointer { tail: a, .. }
, SizeSkeleton
::Pointer { tail: b, .. }
) => {
2054 pub trait HasTyCtxt
<'tcx
>: HasDataLayout
{
2055 fn tcx(&self) -> TyCtxt
<'tcx
>;
2058 pub trait HasParamEnv
<'tcx
> {
2059 fn param_env(&self) -> ty
::ParamEnv
<'tcx
>;
2062 impl<'tcx
> HasDataLayout
for TyCtxt
<'tcx
> {
2064 fn data_layout(&self) -> &TargetDataLayout
{
2069 impl<'tcx
> HasTargetSpec
for TyCtxt
<'tcx
> {
2070 fn target_spec(&self) -> &Target
{
2075 impl<'tcx
> HasTyCtxt
<'tcx
> for TyCtxt
<'tcx
> {
2077 fn tcx(&self) -> TyCtxt
<'tcx
> {
2082 impl<'tcx
> HasDataLayout
for ty
::query
::TyCtxtAt
<'tcx
> {
2084 fn data_layout(&self) -> &TargetDataLayout
{
2089 impl<'tcx
> HasTargetSpec
for ty
::query
::TyCtxtAt
<'tcx
> {
2090 fn target_spec(&self) -> &Target
{
2095 impl<'tcx
> HasTyCtxt
<'tcx
> for ty
::query
::TyCtxtAt
<'tcx
> {
2097 fn tcx(&self) -> TyCtxt
<'tcx
> {
2102 impl<'tcx
, C
> HasParamEnv
<'tcx
> for LayoutCx
<'tcx
, C
> {
2103 fn param_env(&self) -> ty
::ParamEnv
<'tcx
> {
2108 impl<'tcx
, T
: HasDataLayout
> HasDataLayout
for LayoutCx
<'tcx
, T
> {
2109 fn data_layout(&self) -> &TargetDataLayout
{
2110 self.tcx
.data_layout()
2114 impl<'tcx
, T
: HasTargetSpec
> HasTargetSpec
for LayoutCx
<'tcx
, T
> {
2115 fn target_spec(&self) -> &Target
{
2116 self.tcx
.target_spec()
2120 impl<'tcx
, T
: HasTyCtxt
<'tcx
>> HasTyCtxt
<'tcx
> for LayoutCx
<'tcx
, T
> {
2121 fn tcx(&self) -> TyCtxt
<'tcx
> {
2126 pub trait MaybeResult
<T
> {
2129 fn from(x
: Result
<T
, Self::Error
>) -> Self;
2130 fn to_result(self) -> Result
<T
, Self::Error
>;
2133 impl<T
> MaybeResult
<T
> for T
{
2136 fn from(Ok(x
): Result
<T
, Self::Error
>) -> Self {
2139 fn to_result(self) -> Result
<T
, Self::Error
> {
2144 impl<T
, E
> MaybeResult
<T
> for Result
<T
, E
> {
2147 fn from(x
: Result
<T
, Self::Error
>) -> Self {
2150 fn to_result(self) -> Result
<T
, Self::Error
> {
2155 pub type TyAndLayout
<'tcx
> = rustc_target
::abi
::TyAndLayout
<'tcx
, Ty
<'tcx
>>;
2157 /// Trait for contexts that want to be able to compute layouts of types.
2158 /// This automatically gives access to `LayoutOf`, through a blanket `impl`.
2159 pub trait LayoutOfHelpers
<'tcx
>: HasDataLayout
+ HasTyCtxt
<'tcx
> + HasParamEnv
<'tcx
> {
2160 /// The `TyAndLayout`-wrapping type (or `TyAndLayout` itself), which will be
2161 /// returned from `layout_of` (see also `handle_layout_err`).
2162 type LayoutOfResult
: MaybeResult
<TyAndLayout
<'tcx
>>;
2164 /// `Span` to use for `tcx.at(span)`, from `layout_of`.
2165 // FIXME(eddyb) perhaps make this mandatory to get contexts to track it better?
2167 fn layout_tcx_at_span(&self) -> Span
{
2171 /// Helper used for `layout_of`, to adapt `tcx.layout_of(...)` into a
2172 /// `Self::LayoutOfResult` (which does not need to be a `Result<...>`).
2174 /// Most `impl`s, which propagate `LayoutError`s, should simply return `err`,
2175 /// but this hook allows e.g. codegen to return only `TyAndLayout` from its
2176 /// `cx.layout_of(...)`, without any `Result<...>` around it to deal with
2177 /// (and any `LayoutError`s are turned into fatal errors or ICEs).
2178 fn handle_layout_err(
2180 err
: LayoutError
<'tcx
>,
2183 ) -> <Self::LayoutOfResult
as MaybeResult
<TyAndLayout
<'tcx
>>>::Error
;
2186 /// Blanket extension trait for contexts that can compute layouts of types.
2187 pub trait LayoutOf
<'tcx
>: LayoutOfHelpers
<'tcx
> {
2188 /// Computes the layout of a type. Note that this implicitly
2189 /// executes in "reveal all" mode, and will normalize the input type.
2191 fn layout_of(&self, ty
: Ty
<'tcx
>) -> Self::LayoutOfResult
{
2192 self.spanned_layout_of(ty
, DUMMY_SP
)
2195 /// Computes the layout of a type, at `span`. Note that this implicitly
2196 /// executes in "reveal all" mode, and will normalize the input type.
2197 // FIXME(eddyb) avoid passing information like this, and instead add more
2198 // `TyCtxt::at`-like APIs to be able to do e.g. `cx.at(span).layout_of(ty)`.
2200 fn spanned_layout_of(&self, ty
: Ty
<'tcx
>, span
: Span
) -> Self::LayoutOfResult
{
2201 let span
= if !span
.is_dummy() { span }
else { self.layout_tcx_at_span() }
;
2202 let tcx
= self.tcx().at(span
);
2205 tcx
.layout_of(self.param_env().and(ty
))
2206 .map_err(|err
| self.handle_layout_err(err
, span
, ty
)),
2211 impl<'tcx
, C
: LayoutOfHelpers
<'tcx
>> LayoutOf
<'tcx
> for C {}
2213 impl<'tcx
> LayoutOfHelpers
<'tcx
> for LayoutCx
<'tcx
, TyCtxt
<'tcx
>> {
2214 type LayoutOfResult
= Result
<TyAndLayout
<'tcx
>, LayoutError
<'tcx
>>;
2217 fn handle_layout_err(&self, err
: LayoutError
<'tcx
>, _
: Span
, _
: Ty
<'tcx
>) -> LayoutError
<'tcx
> {
2222 impl<'tcx
> LayoutOfHelpers
<'tcx
> for LayoutCx
<'tcx
, ty
::query
::TyCtxtAt
<'tcx
>> {
2223 type LayoutOfResult
= Result
<TyAndLayout
<'tcx
>, LayoutError
<'tcx
>>;
2226 fn layout_tcx_at_span(&self) -> Span
{
2231 fn handle_layout_err(&self, err
: LayoutError
<'tcx
>, _
: Span
, _
: Ty
<'tcx
>) -> LayoutError
<'tcx
> {
2236 impl<'tcx
, C
> TyAbiInterface
<'tcx
, C
> for Ty
<'tcx
>
2238 C
: HasTyCtxt
<'tcx
> + HasParamEnv
<'tcx
>,
2240 fn ty_and_layout_for_variant(
2241 this
: TyAndLayout
<'tcx
>,
2243 variant_index
: VariantIdx
,
2244 ) -> TyAndLayout
<'tcx
> {
2245 let layout
= match this
.variants
{
2246 Variants
::Single { index }
2247 // If all variants but one are uninhabited, the variant layout is the enum layout.
2248 if index
== variant_index
&&
2249 // Don't confuse variants of uninhabited enums with the enum itself.
2250 // For more details see https://github.com/rust-lang/rust/issues/69763.
2251 this
.fields
!= FieldsShape
::Primitive
=>
2256 Variants
::Single { index }
=> {
2258 let param_env
= cx
.param_env();
2260 // Deny calling for_variant more than once for non-Single enums.
2261 if let Ok(original_layout
) = tcx
.layout_of(param_env
.and(this
.ty
)) {
2262 assert_eq
!(original_layout
.variants
, Variants
::Single { index }
);
2265 let fields
= match this
.ty
.kind() {
2266 ty
::Adt(def
, _
) if def
.variants().is_empty() =>
2267 bug
!("for_variant called on zero-variant enum"),
2268 ty
::Adt(def
, _
) => def
.variant(variant_index
).fields
.len(),
2271 tcx
.intern_layout(LayoutS
{
2272 variants
: Variants
::Single { index: variant_index }
,
2273 fields
: match NonZeroUsize
::new(fields
) {
2274 Some(fields
) => FieldsShape
::Union(fields
),
2275 None
=> FieldsShape
::Arbitrary { offsets: vec![], memory_index: vec![] }
,
2277 abi
: Abi
::Uninhabited
,
2278 largest_niche
: None
,
2279 align
: tcx
.data_layout
.i8_align
,
2284 Variants
::Multiple { ref variants, .. }
=> variants
[variant_index
],
2287 assert_eq
!(*layout
.variants(), Variants
::Single { index: variant_index }
);
2289 TyAndLayout { ty: this.ty, layout }
2292 fn ty_and_layout_field(this
: TyAndLayout
<'tcx
>, cx
: &C
, i
: usize) -> TyAndLayout
<'tcx
> {
2293 enum TyMaybeWithLayout
<'tcx
> {
2295 TyAndLayout(TyAndLayout
<'tcx
>),
2298 fn field_ty_or_layout
<'tcx
>(
2299 this
: TyAndLayout
<'tcx
>,
2300 cx
: &(impl HasTyCtxt
<'tcx
> + HasParamEnv
<'tcx
>),
2302 ) -> TyMaybeWithLayout
<'tcx
> {
2304 let tag_layout
= |tag
: Scalar
| -> TyAndLayout
<'tcx
> {
2306 layout
: tcx
.intern_layout(LayoutS
::scalar(cx
, tag
)),
2307 ty
: tag
.value
.to_ty(tcx
),
2311 match *this
.ty
.kind() {
2320 | ty
::GeneratorWitness(..)
2322 | ty
::Dynamic(..) => bug
!("TyAndLayout::field({:?}): not applicable", this
),
2324 // Potentially-fat pointers.
2325 ty
::Ref(_
, pointee
, _
) | ty
::RawPtr(ty
::TypeAndMut { ty: pointee, .. }
) => {
2326 assert
!(i
< this
.fields
.count());
2328 // Reuse the fat `*T` type as its own thin pointer data field.
2329 // This provides information about, e.g., DST struct pointees
2330 // (which may have no non-DST form), and will work as long
2331 // as the `Abi` or `FieldsShape` is checked by users.
2333 let nil
= tcx
.mk_unit();
2334 let unit_ptr_ty
= if this
.ty
.is_unsafe_ptr() {
2337 tcx
.mk_mut_ref(tcx
.lifetimes
.re_static
, nil
)
2340 // NOTE(eddyb) using an empty `ParamEnv`, and `unwrap`-ing
2341 // the `Result` should always work because the type is
2342 // always either `*mut ()` or `&'static mut ()`.
2343 return TyMaybeWithLayout
::TyAndLayout(TyAndLayout
{
2345 ..tcx
.layout_of(ty
::ParamEnv
::reveal_all().and(unit_ptr_ty
)).unwrap()
2349 match tcx
.struct_tail_erasing_lifetimes(pointee
, cx
.param_env()).kind() {
2350 ty
::Slice(_
) | ty
::Str
=> TyMaybeWithLayout
::Ty(tcx
.types
.usize),
2351 ty
::Dynamic(_
, _
) => {
2352 TyMaybeWithLayout
::Ty(tcx
.mk_imm_ref(
2353 tcx
.lifetimes
.re_static
,
2354 tcx
.mk_array(tcx
.types
.usize, 3),
2356 /* FIXME: use actual fn pointers
2357 Warning: naively computing the number of entries in the
2358 vtable by counting the methods on the trait + methods on
2359 all parent traits does not work, because some methods can
2360 be not object safe and thus excluded from the vtable.
2361 Increase this counter if you tried to implement this but
2362 failed to do it without duplicating a lot of code from
2363 other places in the compiler: 2
2365 tcx.mk_array(tcx.types.usize, 3),
2366 tcx.mk_array(Option<fn()>),
2370 _
=> bug
!("TyAndLayout::field({:?}): not applicable", this
),
2374 // Arrays and slices.
2375 ty
::Array(element
, _
) | ty
::Slice(element
) => TyMaybeWithLayout
::Ty(element
),
2376 ty
::Str
=> TyMaybeWithLayout
::Ty(tcx
.types
.u8),
2378 // Tuples, generators and closures.
2379 ty
::Closure(_
, ref substs
) => field_ty_or_layout(
2380 TyAndLayout { ty: substs.as_closure().tupled_upvars_ty(), ..this }
,
2385 ty
::Generator(def_id
, ref substs
, _
) => match this
.variants
{
2386 Variants
::Single { index }
=> TyMaybeWithLayout
::Ty(
2389 .state_tys(def_id
, tcx
)
2390 .nth(index
.as_usize())
2395 Variants
::Multiple { tag, tag_field, .. }
=> {
2397 return TyMaybeWithLayout
::TyAndLayout(tag_layout(tag
));
2399 TyMaybeWithLayout
::Ty(substs
.as_generator().prefix_tys().nth(i
).unwrap())
2403 ty
::Tuple(tys
) => TyMaybeWithLayout
::Ty(tys
[i
]),
2406 ty
::Adt(def
, substs
) => {
2407 match this
.variants
{
2408 Variants
::Single { index }
=> {
2409 TyMaybeWithLayout
::Ty(def
.variant(index
).fields
[i
].ty(tcx
, substs
))
2412 // Discriminant field for enums (where applicable).
2413 Variants
::Multiple { tag, .. }
=> {
2415 return TyMaybeWithLayout
::TyAndLayout(tag_layout(tag
));
2422 | ty
::Placeholder(..)
2426 | ty
::Error(_
) => bug
!("TyAndLayout::field: unexpected type `{}`", this
.ty
),
2430 match field_ty_or_layout(this
, cx
, i
) {
2431 TyMaybeWithLayout
::Ty(field_ty
) => {
2432 cx
.tcx().layout_of(cx
.param_env().and(field_ty
)).unwrap_or_else(|e
| {
2434 "failed to get layout for `{}`: {},\n\
2435 despite it being a field (#{}) of an existing layout: {:#?}",
2443 TyMaybeWithLayout
::TyAndLayout(field_layout
) => field_layout
,
2447 fn ty_and_layout_pointee_info_at(
2448 this
: TyAndLayout
<'tcx
>,
2451 ) -> Option
<PointeeInfo
> {
2453 let param_env
= cx
.param_env();
2455 let addr_space_of_ty
= |ty
: Ty
<'tcx
>| {
2456 if ty
.is_fn() { cx.data_layout().instruction_address_space }
else { AddressSpace::DATA }
2459 let pointee_info
= match *this
.ty
.kind() {
2460 ty
::RawPtr(mt
) if offset
.bytes() == 0 => {
2461 tcx
.layout_of(param_env
.and(mt
.ty
)).ok().map(|layout
| PointeeInfo
{
2463 align
: layout
.align
.abi
,
2465 address_space
: addr_space_of_ty(mt
.ty
),
2468 ty
::FnPtr(fn_sig
) if offset
.bytes() == 0 => {
2469 tcx
.layout_of(param_env
.and(tcx
.mk_fn_ptr(fn_sig
))).ok().map(|layout
| PointeeInfo
{
2471 align
: layout
.align
.abi
,
2473 address_space
: cx
.data_layout().instruction_address_space
,
2476 ty
::Ref(_
, ty
, mt
) if offset
.bytes() == 0 => {
2477 let address_space
= addr_space_of_ty(ty
);
2478 let kind
= if tcx
.sess
.opts
.optimize
== OptLevel
::No
{
2479 // Use conservative pointer kind if not optimizing. This saves us the
2480 // Freeze/Unpin queries, and can save time in the codegen backend (noalias
2481 // attributes in LLVM have compile-time cost even in unoptimized builds).
2485 hir
::Mutability
::Not
=> {
2486 if ty
.is_freeze(tcx
.at(DUMMY_SP
), cx
.param_env()) {
2492 hir
::Mutability
::Mut
=> {
2493 // References to self-referential structures should not be considered
2494 // noalias, as another pointer to the structure can be obtained, that
2495 // is not based-on the original reference. We consider all !Unpin
2496 // types to be potentially self-referential here.
2497 if ty
.is_unpin(tcx
.at(DUMMY_SP
), cx
.param_env()) {
2498 PointerKind
::UniqueBorrowed
2506 tcx
.layout_of(param_env
.and(ty
)).ok().map(|layout
| PointeeInfo
{
2508 align
: layout
.align
.abi
,
2515 let mut data_variant
= match this
.variants
{
2516 // Within the discriminant field, only the niche itself is
2517 // always initialized, so we only check for a pointer at its
2520 // If the niche is a pointer, it's either valid (according
2521 // to its type), or null (which the niche field's scalar
2522 // validity range encodes). This allows using
2523 // `dereferenceable_or_null` for e.g., `Option<&T>`, and
2524 // this will continue to work as long as we don't start
2525 // using more niches than just null (e.g., the first page of
2526 // the address space, or unaligned pointers).
2527 Variants
::Multiple
{
2528 tag_encoding
: TagEncoding
::Niche { dataful_variant, .. }
,
2531 } if this
.fields
.offset(tag_field
) == offset
=> {
2532 Some(this
.for_variant(cx
, dataful_variant
))
2537 if let Some(variant
) = data_variant
{
2538 // We're not interested in any unions.
2539 if let FieldsShape
::Union(_
) = variant
.fields
{
2540 data_variant
= None
;
2544 let mut result
= None
;
2546 if let Some(variant
) = data_variant
{
2547 let ptr_end
= offset
+ Pointer
.size(cx
);
2548 for i
in 0..variant
.fields
.count() {
2549 let field_start
= variant
.fields
.offset(i
);
2550 if field_start
<= offset
{
2551 let field
= variant
.field(cx
, i
);
2552 result
= field
.to_result().ok().and_then(|field
| {
2553 if ptr_end
<= field_start
+ field
.size
{
2554 // We found the right field, look inside it.
2556 field
.pointee_info_at(cx
, offset
- field_start
);
2562 if result
.is_some() {
2569 // FIXME(eddyb) This should be for `ptr::Unique<T>`, not `Box<T>`.
2570 if let Some(ref mut pointee
) = result
{
2571 if let ty
::Adt(def
, _
) = this
.ty
.kind() {
2572 if def
.is_box() && offset
.bytes() == 0 {
2573 pointee
.safe
= Some(PointerKind
::UniqueOwned
);
2583 "pointee_info_at (offset={:?}, type kind: {:?}) => {:?}",
2593 impl<'tcx
> ty
::Instance
<'tcx
> {
2594 // NOTE(eddyb) this is private to avoid using it from outside of
2595 // `fn_abi_of_instance` - any other uses are either too high-level
2596 // for `Instance` (e.g. typeck would use `Ty::fn_sig` instead),
2597 // or should go through `FnAbi` instead, to avoid losing any
2598 // adjustments `fn_abi_of_instance` might be performing.
2599 fn fn_sig_for_fn_abi(
2602 param_env
: ty
::ParamEnv
<'tcx
>,
2603 ) -> ty
::PolyFnSig
<'tcx
> {
2604 let ty
= self.ty(tcx
, param_env
);
2607 // HACK(davidtwco,eddyb): This is a workaround for polymorphization considering
2608 // parameters unused if they show up in the signature, but not in the `mir::Body`
2609 // (i.e. due to being inside a projection that got normalized, see
2610 // `src/test/ui/polymorphization/normalized_sig_types.rs`), and codegen not keeping
2611 // track of a polymorphization `ParamEnv` to allow normalizing later.
2612 let mut sig
= match *ty
.kind() {
2613 ty
::FnDef(def_id
, substs
) => tcx
2614 .normalize_erasing_regions(tcx
.param_env(def_id
), tcx
.fn_sig(def_id
))
2615 .subst(tcx
, substs
),
2616 _
=> unreachable
!(),
2619 if let ty
::InstanceDef
::VtableShim(..) = self.def
{
2620 // Modify `fn(self, ...)` to `fn(self: *mut Self, ...)`.
2621 sig
= sig
.map_bound(|mut sig
| {
2622 let mut inputs_and_output
= sig
.inputs_and_output
.to_vec();
2623 inputs_and_output
[0] = tcx
.mk_mut_ptr(inputs_and_output
[0]);
2624 sig
.inputs_and_output
= tcx
.intern_type_list(&inputs_and_output
);
2630 ty
::Closure(def_id
, substs
) => {
2631 let sig
= substs
.as_closure().sig();
2633 let bound_vars
= tcx
.mk_bound_variable_kinds(
2636 .chain(iter
::once(ty
::BoundVariableKind
::Region(ty
::BrEnv
))),
2638 let br
= ty
::BoundRegion
{
2639 var
: ty
::BoundVar
::from_usize(bound_vars
.len() - 1),
2640 kind
: ty
::BoundRegionKind
::BrEnv
,
2642 let env_region
= ty
::ReLateBound(ty
::INNERMOST
, br
);
2643 let env_ty
= tcx
.closure_env_ty(def_id
, substs
, env_region
).unwrap();
2645 let sig
= sig
.skip_binder();
2646 ty
::Binder
::bind_with_vars(
2648 iter
::once(env_ty
).chain(sig
.inputs().iter().cloned()),
2657 ty
::Generator(_
, substs
, _
) => {
2658 let sig
= substs
.as_generator().poly_sig();
2660 let bound_vars
= tcx
.mk_bound_variable_kinds(
2663 .chain(iter
::once(ty
::BoundVariableKind
::Region(ty
::BrEnv
))),
2665 let br
= ty
::BoundRegion
{
2666 var
: ty
::BoundVar
::from_usize(bound_vars
.len() - 1),
2667 kind
: ty
::BoundRegionKind
::BrEnv
,
2669 let env_region
= ty
::ReLateBound(ty
::INNERMOST
, br
);
2670 let env_ty
= tcx
.mk_mut_ref(tcx
.mk_region(env_region
), ty
);
2672 let pin_did
= tcx
.require_lang_item(LangItem
::Pin
, None
);
2673 let pin_adt_ref
= tcx
.adt_def(pin_did
);
2674 let pin_substs
= tcx
.intern_substs(&[env_ty
.into()]);
2675 let env_ty
= tcx
.mk_adt(pin_adt_ref
, pin_substs
);
2677 let sig
= sig
.skip_binder();
2678 let state_did
= tcx
.require_lang_item(LangItem
::GeneratorState
, None
);
2679 let state_adt_ref
= tcx
.adt_def(state_did
);
2680 let state_substs
= tcx
.intern_substs(&[sig
.yield_ty
.into(), sig
.return_ty
.into()]);
2681 let ret_ty
= tcx
.mk_adt(state_adt_ref
, state_substs
);
2682 ty
::Binder
::bind_with_vars(
2684 [env_ty
, sig
.resume_ty
].iter(),
2687 hir
::Unsafety
::Normal
,
2688 rustc_target
::spec
::abi
::Abi
::Rust
,
2693 _
=> bug
!("unexpected type {:?} in Instance::fn_sig", ty
),
2698 /// Calculates whether a function's ABI can unwind or not.
2700 /// This takes two primary parameters:
2702 /// * `codegen_fn_attr_flags` - these are flags calculated as part of the
2703 /// codegen attrs for a defined function. For function pointers this set of
2704 /// flags is the empty set. This is only applicable for Rust-defined
2705 /// functions, and generally isn't needed except for small optimizations where
2706 /// we try to say a function which otherwise might look like it could unwind
2707 /// doesn't actually unwind (such as for intrinsics and such).
2709 /// * `abi` - this is the ABI that the function is defined with. This is the
2710 /// primary factor for determining whether a function can unwind or not.
2712 /// Note that in this case unwinding is not necessarily panicking in Rust. Rust
2713 /// panics are implemented with unwinds on most platform (when
2714 /// `-Cpanic=unwind`), but this also accounts for `-Cpanic=abort` build modes.
2715 /// Notably unwinding is disallowed for more non-Rust ABIs unless it's
2716 /// specifically in the name (e.g. `"C-unwind"`). Unwinding within each ABI is
2717 /// defined for each ABI individually, but it always corresponds to some form of
2718 /// stack-based unwinding (the exact mechanism of which varies
2719 /// platform-by-platform).
2721 /// Rust functions are classified whether or not they can unwind based on the
2722 /// active "panic strategy". In other words Rust functions are considered to
2723 /// unwind in `-Cpanic=unwind` mode and cannot unwind in `-Cpanic=abort` mode.
2724 /// Note that Rust supports intermingling panic=abort and panic=unwind code, but
2725 /// only if the final panic mode is panic=abort. In this scenario any code
2726 /// previously compiled assuming that a function can unwind is still correct, it
2727 /// just never happens to actually unwind at runtime.
2729 /// This function's answer to whether or not a function can unwind is quite
2730 /// impactful throughout the compiler. This affects things like:
2732 /// * Calling a function which can't unwind means codegen simply ignores any
2733 /// associated unwinding cleanup.
2734 /// * Calling a function which can unwind from a function which can't unwind
2735 /// causes the `abort_unwinding_calls` MIR pass to insert a landing pad that
2736 /// aborts the process.
2737 /// * This affects whether functions have the LLVM `nounwind` attribute, which
2738 /// affects various optimizations and codegen.
2740 /// FIXME: this is actually buggy with respect to Rust functions. Rust functions
2741 /// compiled with `-Cpanic=unwind` and referenced from another crate compiled
2742 /// with `-Cpanic=abort` will look like they can't unwind when in fact they
2743 /// might (from a foreign exception or similar).
2745 pub fn fn_can_unwind
<'tcx
>(
2747 codegen_fn_attr_flags
: CodegenFnAttrFlags
,
2750 // Special attribute for functions which can't unwind.
2751 if codegen_fn_attr_flags
.contains(CodegenFnAttrFlags
::NEVER_UNWIND
) {
2755 // Otherwise if this isn't special then unwinding is generally determined by
2756 // the ABI of the itself. ABIs like `C` have variants which also
2757 // specifically allow unwinding (`C-unwind`), but not all platform-specific
2758 // ABIs have such an option. Otherwise the only other thing here is Rust
2759 // itself, and those ABIs are determined by the panic strategy configured
2760 // for this compilation.
2762 // Unfortunately at this time there's also another caveat. Rust [RFC
2763 // 2945][rfc] has been accepted and is in the process of being implemented
2764 // and stabilized. In this interim state we need to deal with historical
2765 // rustc behavior as well as plan for future rustc behavior.
2767 // Historically functions declared with `extern "C"` were marked at the
2768 // codegen layer as `nounwind`. This happened regardless of `panic=unwind`
2769 // or not. This is UB for functions in `panic=unwind` mode that then
2770 // actually panic and unwind. Note that this behavior is true for both
2771 // externally declared functions as well as Rust-defined function.
2773 // To fix this UB rustc would like to change in the future to catch unwinds
2774 // from function calls that may unwind within a Rust-defined `extern "C"`
2775 // function and forcibly abort the process, thereby respecting the
2776 // `nounwind` attribute emitted for `extern "C"`. This behavior change isn't
2777 // ready to roll out, so determining whether or not the `C` family of ABIs
2778 // unwinds is conditional not only on their definition but also whether the
2779 // `#![feature(c_unwind)]` feature gate is active.
2781 // Note that this means that unlike historical compilers rustc now, by
2782 // default, unconditionally thinks that the `C` ABI may unwind. This will
2783 // prevent some optimization opportunities, however, so we try to scope this
2784 // change and only assume that `C` unwinds with `panic=unwind` (as opposed
2785 // to `panic=abort`).
2787 // Eventually the check against `c_unwind` here will ideally get removed and
2788 // this'll be a little cleaner as it'll be a straightforward check of the
2791 // [rfc]: https://github.com/rust-lang/rfcs/blob/master/text/2945-c-unwind-abi.md
2797 | Stdcall { unwind }
2798 | Fastcall { unwind }
2799 | Vectorcall { unwind }
2800 | Thiscall { unwind }
2803 | SysV64 { unwind }
=> {
2805 || (!tcx
.features().c_unwind
&& tcx
.sess
.panic_strategy() == PanicStrategy
::Unwind
)
2813 | AvrNonBlockingInterrupt
2814 | CCmseNonSecureCall
2818 | Unadjusted
=> false,
2819 Rust
| RustCall
=> tcx
.sess
.panic_strategy() == PanicStrategy
::Unwind
,
2824 pub fn conv_from_spec_abi(tcx
: TyCtxt
<'_
>, abi
: SpecAbi
) -> Conv
{
2825 use rustc_target
::spec
::abi
::Abi
::*;
2826 match tcx
.sess
.target
.adjust_abi(abi
) {
2827 RustIntrinsic
| PlatformIntrinsic
| Rust
| RustCall
=> Conv
::Rust
,
2829 // It's the ABI's job to select this, not ours.
2830 System { .. }
=> bug
!("system abi should be selected elsewhere"),
2831 EfiApi
=> bug
!("eficall abi should be selected elsewhere"),
2833 Stdcall { .. }
=> Conv
::X86Stdcall
,
2834 Fastcall { .. }
=> Conv
::X86Fastcall
,
2835 Vectorcall { .. }
=> Conv
::X86VectorCall
,
2836 Thiscall { .. }
=> Conv
::X86ThisCall
,
2837 C { .. }
=> Conv
::C
,
2838 Unadjusted
=> Conv
::C
,
2839 Win64 { .. }
=> Conv
::X86_64Win64
,
2840 SysV64 { .. }
=> Conv
::X86_64SysV
,
2841 Aapcs { .. }
=> Conv
::ArmAapcs
,
2842 CCmseNonSecureCall
=> Conv
::CCmseNonSecureCall
,
2843 PtxKernel
=> Conv
::PtxKernel
,
2844 Msp430Interrupt
=> Conv
::Msp430Intr
,
2845 X86Interrupt
=> Conv
::X86Intr
,
2846 AmdGpuKernel
=> Conv
::AmdGpuKernel
,
2847 AvrInterrupt
=> Conv
::AvrInterrupt
,
2848 AvrNonBlockingInterrupt
=> Conv
::AvrNonBlockingInterrupt
,
2851 // These API constants ought to be more specific...
2852 Cdecl { .. }
=> Conv
::C
,
2856 /// Error produced by attempting to compute or adjust a `FnAbi`.
2857 #[derive(Copy, Clone, Debug, HashStable)]
2858 pub enum FnAbiError
<'tcx
> {
2859 /// Error produced by a `layout_of` call, while computing `FnAbi` initially.
2860 Layout(LayoutError
<'tcx
>),
2862 /// Error produced by attempting to adjust a `FnAbi`, for a "foreign" ABI.
2863 AdjustForForeignAbi(call
::AdjustForForeignAbiError
),
2866 impl<'tcx
> From
<LayoutError
<'tcx
>> for FnAbiError
<'tcx
> {
2867 fn from(err
: LayoutError
<'tcx
>) -> Self {
2872 impl From
<call
::AdjustForForeignAbiError
> for FnAbiError
<'_
> {
2873 fn from(err
: call
::AdjustForForeignAbiError
) -> Self {
2874 Self::AdjustForForeignAbi(err
)
2878 impl<'tcx
> fmt
::Display
for FnAbiError
<'tcx
> {
2879 fn fmt(&self, f
: &mut fmt
::Formatter
<'_
>) -> fmt
::Result
{
2881 Self::Layout(err
) => err
.fmt(f
),
2882 Self::AdjustForForeignAbi(err
) => err
.fmt(f
),
2887 // FIXME(eddyb) maybe use something like this for an unified `fn_abi_of`, not
2888 // just for error handling.
2890 pub enum FnAbiRequest
<'tcx
> {
2891 OfFnPtr { sig: ty::PolyFnSig<'tcx>, extra_args: &'tcx ty::List<Ty<'tcx>> }
,
2892 OfInstance { instance: ty::Instance<'tcx>, extra_args: &'tcx ty::List<Ty<'tcx>> }
,
2895 /// Trait for contexts that want to be able to compute `FnAbi`s.
2896 /// This automatically gives access to `FnAbiOf`, through a blanket `impl`.
2897 pub trait FnAbiOfHelpers
<'tcx
>: LayoutOfHelpers
<'tcx
> {
2898 /// The `&FnAbi`-wrapping type (or `&FnAbi` itself), which will be
2899 /// returned from `fn_abi_of_*` (see also `handle_fn_abi_err`).
2900 type FnAbiOfResult
: MaybeResult
<&'tcx FnAbi
<'tcx
, Ty
<'tcx
>>>;
2902 /// Helper used for `fn_abi_of_*`, to adapt `tcx.fn_abi_of_*(...)` into a
2903 /// `Self::FnAbiOfResult` (which does not need to be a `Result<...>`).
2905 /// Most `impl`s, which propagate `FnAbiError`s, should simply return `err`,
2906 /// but this hook allows e.g. codegen to return only `&FnAbi` from its
2907 /// `cx.fn_abi_of_*(...)`, without any `Result<...>` around it to deal with
2908 /// (and any `FnAbiError`s are turned into fatal errors or ICEs).
2909 fn handle_fn_abi_err(
2911 err
: FnAbiError
<'tcx
>,
2913 fn_abi_request
: FnAbiRequest
<'tcx
>,
2914 ) -> <Self::FnAbiOfResult
as MaybeResult
<&'tcx FnAbi
<'tcx
, Ty
<'tcx
>>>>::Error
;
2917 /// Blanket extension trait for contexts that can compute `FnAbi`s.
2918 pub trait FnAbiOf
<'tcx
>: FnAbiOfHelpers
<'tcx
> {
2919 /// Compute a `FnAbi` suitable for indirect calls, i.e. to `fn` pointers.
2921 /// NB: this doesn't handle virtual calls - those should use `fn_abi_of_instance`
2922 /// instead, where the instance is an `InstanceDef::Virtual`.
2924 fn fn_abi_of_fn_ptr(
2926 sig
: ty
::PolyFnSig
<'tcx
>,
2927 extra_args
: &'tcx ty
::List
<Ty
<'tcx
>>,
2928 ) -> Self::FnAbiOfResult
{
2929 // FIXME(eddyb) get a better `span` here.
2930 let span
= self.layout_tcx_at_span();
2931 let tcx
= self.tcx().at(span
);
2933 MaybeResult
::from(tcx
.fn_abi_of_fn_ptr(self.param_env().and((sig
, extra_args
))).map_err(
2934 |err
| self.handle_fn_abi_err(err
, span
, FnAbiRequest
::OfFnPtr { sig, extra_args }
),
2938 /// Compute a `FnAbi` suitable for declaring/defining an `fn` instance, and for
2939 /// direct calls to an `fn`.
2941 /// NB: that includes virtual calls, which are represented by "direct calls"
2942 /// to an `InstanceDef::Virtual` instance (of `<dyn Trait as Trait>::fn`).
2944 fn fn_abi_of_instance(
2946 instance
: ty
::Instance
<'tcx
>,
2947 extra_args
: &'tcx ty
::List
<Ty
<'tcx
>>,
2948 ) -> Self::FnAbiOfResult
{
2949 // FIXME(eddyb) get a better `span` here.
2950 let span
= self.layout_tcx_at_span();
2951 let tcx
= self.tcx().at(span
);
2954 tcx
.fn_abi_of_instance(self.param_env().and((instance
, extra_args
))).map_err(|err
| {
2955 // HACK(eddyb) at least for definitions of/calls to `Instance`s,
2956 // we can get some kind of span even if one wasn't provided.
2957 // However, we don't do this early in order to avoid calling
2958 // `def_span` unconditionally (which may have a perf penalty).
2959 let span
= if !span
.is_dummy() { span }
else { tcx.def_span(instance.def_id()) }
;
2960 self.handle_fn_abi_err(err
, span
, FnAbiRequest
::OfInstance { instance, extra_args }
)
2966 impl<'tcx
, C
: FnAbiOfHelpers
<'tcx
>> FnAbiOf
<'tcx
> for C {}
2968 fn fn_abi_of_fn_ptr
<'tcx
>(
2970 query
: ty
::ParamEnvAnd
<'tcx
, (ty
::PolyFnSig
<'tcx
>, &'tcx ty
::List
<Ty
<'tcx
>>)>,
2971 ) -> Result
<&'tcx FnAbi
<'tcx
, Ty
<'tcx
>>, FnAbiError
<'tcx
>> {
2972 let (param_env
, (sig
, extra_args
)) = query
.into_parts();
2974 LayoutCx { tcx, param_env }
.fn_abi_new_uncached(
2978 CodegenFnAttrFlags
::empty(),
2983 fn fn_abi_of_instance
<'tcx
>(
2985 query
: ty
::ParamEnvAnd
<'tcx
, (ty
::Instance
<'tcx
>, &'tcx ty
::List
<Ty
<'tcx
>>)>,
2986 ) -> Result
<&'tcx FnAbi
<'tcx
, Ty
<'tcx
>>, FnAbiError
<'tcx
>> {
2987 let (param_env
, (instance
, extra_args
)) = query
.into_parts();
2989 let sig
= instance
.fn_sig_for_fn_abi(tcx
, param_env
);
2991 let caller_location
= if instance
.def
.requires_caller_location(tcx
) {
2992 Some(tcx
.caller_location_ty())
2997 let attrs
= tcx
.codegen_fn_attrs(instance
.def_id()).flags
;
2999 LayoutCx { tcx, param_env }
.fn_abi_new_uncached(
3004 matches
!(instance
.def
, ty
::InstanceDef
::Virtual(..)),
3008 impl<'tcx
> LayoutCx
<'tcx
, TyCtxt
<'tcx
>> {
3009 // FIXME(eddyb) perhaps group the signature/type-containing (or all of them?)
3010 // arguments of this method, into a separate `struct`.
3011 fn fn_abi_new_uncached(
3013 sig
: ty
::PolyFnSig
<'tcx
>,
3014 extra_args
: &[Ty
<'tcx
>],
3015 caller_location
: Option
<Ty
<'tcx
>>,
3016 codegen_fn_attr_flags
: CodegenFnAttrFlags
,
3017 // FIXME(eddyb) replace this with something typed, like an `enum`.
3018 force_thin_self_ptr
: bool
,
3019 ) -> Result
<&'tcx FnAbi
<'tcx
, Ty
<'tcx
>>, FnAbiError
<'tcx
>> {
3020 debug
!("fn_abi_new_uncached({:?}, {:?})", sig
, extra_args
);
3022 let sig
= self.tcx
.normalize_erasing_late_bound_regions(self.param_env
, sig
);
3024 let conv
= conv_from_spec_abi(self.tcx(), sig
.abi
);
3026 let mut inputs
= sig
.inputs();
3027 let extra_args
= if sig
.abi
== RustCall
{
3028 assert
!(!sig
.c_variadic
&& extra_args
.is_empty());
3030 if let Some(input
) = sig
.inputs().last() {
3031 if let ty
::Tuple(tupled_arguments
) = input
.kind() {
3032 inputs
= &sig
.inputs()[0..sig
.inputs().len() - 1];
3036 "argument to function with \"rust-call\" ABI \
3042 "argument to function with \"rust-call\" ABI \
3047 assert
!(sig
.c_variadic
|| extra_args
.is_empty());
3051 let target
= &self.tcx
.sess
.target
;
3052 let target_env_gnu_like
= matches
!(&target
.env
[..], "gnu" | "musl" | "uclibc");
3053 let win_x64_gnu
= target
.os
== "windows" && target
.arch
== "x86_64" && target
.env
== "gnu";
3054 let linux_s390x_gnu_like
=
3055 target
.os
== "linux" && target
.arch
== "s390x" && target_env_gnu_like
;
3056 let linux_sparc64_gnu_like
=
3057 target
.os
== "linux" && target
.arch
== "sparc64" && target_env_gnu_like
;
3058 let linux_powerpc_gnu_like
=
3059 target
.os
== "linux" && target
.arch
== "powerpc" && target_env_gnu_like
;
3061 let rust_abi
= matches
!(sig
.abi
, RustIntrinsic
| PlatformIntrinsic
| Rust
| RustCall
);
3063 // Handle safe Rust thin and fat pointers.
3064 let adjust_for_rust_scalar
= |attrs
: &mut ArgAttributes
,
3066 layout
: TyAndLayout
<'tcx
>,
3069 // Booleans are always a noundef i1 that needs to be zero-extended.
3070 if scalar
.is_bool() {
3071 attrs
.ext(ArgExtension
::Zext
);
3072 attrs
.set(ArgAttribute
::NoUndef
);
3076 // Scalars which have invalid values cannot be undef.
3077 if !scalar
.is_always_valid(self) {
3078 attrs
.set(ArgAttribute
::NoUndef
);
3081 // Only pointer types handled below.
3082 if scalar
.value
!= Pointer
{
3086 if !scalar
.valid_range
.contains(0) {
3087 attrs
.set(ArgAttribute
::NonNull
);
3090 if let Some(pointee
) = layout
.pointee_info_at(self, offset
) {
3091 if let Some(kind
) = pointee
.safe
{
3092 attrs
.pointee_align
= Some(pointee
.align
);
3094 // `Box` (`UniqueBorrowed`) are not necessarily dereferenceable
3095 // for the entire duration of the function as they can be deallocated
3096 // at any time. Set their valid size to 0.
3097 attrs
.pointee_size
= match kind
{
3098 PointerKind
::UniqueOwned
=> Size
::ZERO
,
3102 // `Box`, `&T`, and `&mut T` cannot be undef.
3103 // Note that this only applies to the value of the pointer itself;
3104 // this attribute doesn't make it UB for the pointed-to data to be undef.
3105 attrs
.set(ArgAttribute
::NoUndef
);
3107 // `Box` pointer parameters never alias because ownership is transferred
3108 // `&mut` pointer parameters never alias other parameters,
3109 // or mutable global data
3111 // `&T` where `T` contains no `UnsafeCell<U>` is immutable,
3112 // and can be marked as both `readonly` and `noalias`, as
3113 // LLVM's definition of `noalias` is based solely on memory
3114 // dependencies rather than pointer equality
3116 // Due to past miscompiles in LLVM, we apply a separate NoAliasMutRef attribute
3117 // for UniqueBorrowed arguments, so that the codegen backend can decide whether
3118 // or not to actually emit the attribute. It can also be controlled with the
3119 // `-Zmutable-noalias` debugging option.
3120 let no_alias
= match kind
{
3121 PointerKind
::Shared
| PointerKind
::UniqueBorrowed
=> false,
3122 PointerKind
::UniqueOwned
=> true,
3123 PointerKind
::Frozen
=> !is_return
,
3126 attrs
.set(ArgAttribute
::NoAlias
);
3129 if kind
== PointerKind
::Frozen
&& !is_return
{
3130 attrs
.set(ArgAttribute
::ReadOnly
);
3133 if kind
== PointerKind
::UniqueBorrowed
&& !is_return
{
3134 attrs
.set(ArgAttribute
::NoAliasMutRef
);
3140 let arg_of
= |ty
: Ty
<'tcx
>, arg_idx
: Option
<usize>| -> Result
<_
, FnAbiError
<'tcx
>> {
3141 let is_return
= arg_idx
.is_none();
3143 let layout
= self.layout_of(ty
)?
;
3144 let layout
= if force_thin_self_ptr
&& arg_idx
== Some(0) {
3145 // Don't pass the vtable, it's not an argument of the virtual fn.
3146 // Instead, pass just the data pointer, but give it the type `*const/mut dyn Trait`
3147 // or `&/&mut dyn Trait` because this is special-cased elsewhere in codegen
3148 make_thin_self_ptr(self, layout
)
3153 let mut arg
= ArgAbi
::new(self, layout
, |layout
, scalar
, offset
| {
3154 let mut attrs
= ArgAttributes
::new();
3155 adjust_for_rust_scalar(&mut attrs
, scalar
, *layout
, offset
, is_return
);
3159 if arg
.layout
.is_zst() {
3160 // For some forsaken reason, x86_64-pc-windows-gnu
3161 // doesn't ignore zero-sized struct arguments.
3162 // The same is true for {s390x,sparc64,powerpc}-unknown-linux-{gnu,musl,uclibc}.
3166 && !linux_s390x_gnu_like
3167 && !linux_sparc64_gnu_like
3168 && !linux_powerpc_gnu_like
)
3170 arg
.mode
= PassMode
::Ignore
;
3177 let mut fn_abi
= FnAbi
{
3178 ret
: arg_of(sig
.output(), None
)?
,
3182 .chain(extra_args
.iter().copied())
3183 .chain(caller_location
)
3185 .map(|(i
, ty
)| arg_of(ty
, Some(i
)))
3186 .collect
::<Result
<_
, _
>>()?
,
3187 c_variadic
: sig
.c_variadic
,
3188 fixed_count
: inputs
.len(),
3190 can_unwind
: fn_can_unwind(self.tcx(), codegen_fn_attr_flags
, sig
.abi
),
3192 self.fn_abi_adjust_for_abi(&mut fn_abi
, sig
.abi
)?
;
3193 debug
!("fn_abi_new_uncached = {:?}", fn_abi
);
3194 Ok(self.tcx
.arena
.alloc(fn_abi
))
3197 fn fn_abi_adjust_for_abi(
3199 fn_abi
: &mut FnAbi
<'tcx
, Ty
<'tcx
>>,
3201 ) -> Result
<(), FnAbiError
<'tcx
>> {
3202 if abi
== SpecAbi
::Unadjusted
{
3206 if abi
== SpecAbi
::Rust
3207 || abi
== SpecAbi
::RustCall
3208 || abi
== SpecAbi
::RustIntrinsic
3209 || abi
== SpecAbi
::PlatformIntrinsic
3211 let fixup
= |arg
: &mut ArgAbi
<'tcx
, Ty
<'tcx
>>| {
3212 if arg
.is_ignore() {
3216 match arg
.layout
.abi
{
3217 Abi
::Aggregate { .. }
=> {}
3219 // This is a fun case! The gist of what this is doing is
3220 // that we want callers and callees to always agree on the
3221 // ABI of how they pass SIMD arguments. If we were to *not*
3222 // make these arguments indirect then they'd be immediates
3223 // in LLVM, which means that they'd used whatever the
3224 // appropriate ABI is for the callee and the caller. That
3225 // means, for example, if the caller doesn't have AVX
3226 // enabled but the callee does, then passing an AVX argument
3227 // across this boundary would cause corrupt data to show up.
3229 // This problem is fixed by unconditionally passing SIMD
3230 // arguments through memory between callers and callees
3231 // which should get them all to agree on ABI regardless of
3232 // target feature sets. Some more information about this
3233 // issue can be found in #44367.
3235 // Note that the platform intrinsic ABI is exempt here as
3236 // that's how we connect up to LLVM and it's unstable
3237 // anyway, we control all calls to it in libstd.
3239 if abi
!= SpecAbi
::PlatformIntrinsic
3240 && self.tcx
.sess
.target
.simd_types_indirect
=>
3242 arg
.make_indirect();
3249 let size
= arg
.layout
.size
;
3250 if arg
.layout
.is_unsized() || size
> Pointer
.size(self) {
3251 arg
.make_indirect();
3253 // We want to pass small aggregates as immediates, but using
3254 // a LLVM aggregate type for this leads to bad optimizations,
3255 // so we pick an appropriately sized integer type instead.
3256 arg
.cast_to(Reg { kind: RegKind::Integer, size }
);
3259 fixup(&mut fn_abi
.ret
);
3260 for arg
in &mut fn_abi
.args
{
3264 fn_abi
.adjust_for_foreign_abi(self, abi
)?
;
3271 fn make_thin_self_ptr
<'tcx
>(
3272 cx
: &(impl HasTyCtxt
<'tcx
> + HasParamEnv
<'tcx
>),
3273 layout
: TyAndLayout
<'tcx
>,
3274 ) -> TyAndLayout
<'tcx
> {
3276 let fat_pointer_ty
= if layout
.is_unsized() {
3277 // unsized `self` is passed as a pointer to `self`
3278 // FIXME (mikeyhew) change this to use &own if it is ever added to the language
3279 tcx
.mk_mut_ptr(layout
.ty
)
3282 Abi
::ScalarPair(..) => (),
3283 _
=> bug
!("receiver type has unsupported layout: {:?}", layout
),
3286 // In the case of Rc<Self>, we need to explicitly pass a *mut RcBox<Self>
3287 // with a Scalar (not ScalarPair) ABI. This is a hack that is understood
3288 // elsewhere in the compiler as a method on a `dyn Trait`.
3289 // To get the type `*mut RcBox<Self>`, we just keep unwrapping newtypes until we
3290 // get a built-in pointer type
3291 let mut fat_pointer_layout
= layout
;
3292 'descend_newtypes
: while !fat_pointer_layout
.ty
.is_unsafe_ptr()
3293 && !fat_pointer_layout
.ty
.is_region_ptr()
3295 for i
in 0..fat_pointer_layout
.fields
.count() {
3296 let field_layout
= fat_pointer_layout
.field(cx
, i
);
3298 if !field_layout
.is_zst() {
3299 fat_pointer_layout
= field_layout
;
3300 continue 'descend_newtypes
;
3304 bug
!("receiver has no non-zero-sized fields {:?}", fat_pointer_layout
);
3307 fat_pointer_layout
.ty
3310 // we now have a type like `*mut RcBox<dyn Trait>`
3311 // change its layout to that of `*mut ()`, a thin pointer, but keep the same type
3312 // this is understood as a special case elsewhere in the compiler
3313 let unit_ptr_ty
= tcx
.mk_mut_ptr(tcx
.mk_unit());
3318 // NOTE(eddyb) using an empty `ParamEnv`, and `unwrap`-ing the `Result`
3319 // should always work because the type is always `*mut ()`.
3320 ..tcx
.layout_of(ty
::ParamEnv
::reveal_all().and(unit_ptr_ty
)).unwrap()