[rustc.git] / compiler / rustc_trait_selection / src / traits / vtable.rs

use crate::errors::DumpVTableEntries;
use crate::traits::{impossible_predicates, is_vtable_safe_method};
use rustc_hir::def_id::DefId;
use rustc_hir::lang_items::LangItem;
use rustc_infer::traits::util::PredicateSet;
use rustc_infer::traits::ImplSource;
use rustc_middle::ty::visit::TypeVisitable;
use rustc_middle::ty::InternalSubsts;
use rustc_middle::ty::{self, GenericParamDefKind, ToPredicate, Ty, TyCtxt, VtblEntry};
use rustc_span::{sym, Span};
use smallvec::SmallVec;

use std::fmt::Debug;
use std::ops::ControlFlow;

#[derive(Clone, Debug)]
pub(super) enum VtblSegment<'tcx> {
    MetadataDSA,
    TraitOwnEntries { trait_ref: ty::PolyTraitRef<'tcx>, emit_vptr: bool },
}

/// Prepare the segments for a vtable
pub(super) fn prepare_vtable_segments<'tcx, T>(
    tcx: TyCtxt<'tcx>,
    trait_ref: ty::PolyTraitRef<'tcx>,
    mut segment_visitor: impl FnMut(VtblSegment<'tcx>) -> ControlFlow<T>,
) -> Option<T> {
    // The following constraints holds for the final arrangement.
    // 1. The whole virtual table of the first direct super trait is included as the
    //    the prefix. If this trait doesn't have any super traits, then this step
    //    consists of the dsa metadata.
    // 2. Then comes the proper pointer metadata(vptr) and all own methods for all
    //    other super traits except those already included as part of the first
    //    direct super trait virtual table.
    // 3. finally, the own methods of this trait.

    // This has the advantage that trait upcasting to the first direct super trait on each level
    // is zero cost, and to another trait includes only replacing the pointer with one level indirection,
    // while not using too much extra memory.

    // For a single inheritance relationship like this,
    //   D --> C --> B --> A
    // The resulting vtable will consists of these segments:
    //  DSA, A, B, C, D

    // For a multiple inheritance relationship like this,
    //   D --> C --> A
    //           \-> B
    // The resulting vtable will consists of these segments:
    //  DSA, A, B, B-vptr, C, D

    // For a diamond inheritance relationship like this,
    //   D --> B --> A
    //     \-> C -/
    // The resulting vtable will consists of these segments:
    //  DSA, A, B, C, C-vptr, D

    // For a more complex inheritance relationship like this:
    //   O --> G --> C --> A
    //     \     \     \-> B
    //     |     |-> F --> D
    //     |           \-> E
    //     |-> N --> J --> H
    //           \     \-> I
    //           |-> M --> K
    //                 \-> L
    // The resulting vtable will consists of these segments:
    //  DSA, A, B, B-vptr, C, D, D-vptr, E, E-vptr, F, F-vptr, G,
    //  H, H-vptr, I, I-vptr, J, J-vptr, K, K-vptr, L, L-vptr, M, M-vptr,
    //  N, N-vptr, O

    // emit dsa segment first.
    if let ControlFlow::Break(v) = (segment_visitor)(VtblSegment::MetadataDSA) {
        return Some(v);
    }

    let mut emit_vptr_on_new_entry = false;
    let mut visited = PredicateSet::new(tcx);
    let predicate = trait_ref.without_const().to_predicate(tcx);
    let mut stack: SmallVec<[(ty::PolyTraitRef<'tcx>, _, _); 5]> =
        smallvec![(trait_ref, emit_vptr_on_new_entry, None)];
    visited.insert(predicate);

    // the main traversal loop:
    // basically we want to cut the inheritance directed graph into a few non-overlapping slices of nodes
    // that each node is emitted after all its descendents have been emitted.
    // so we convert the directed graph into a tree by skipping all previously visited nodes using a visited set.
    // this is done on the fly.
    // Each loop run emits a slice - it starts by find a "childless" unvisited node, backtracking upwards, and it
    // stops after it finds a node that has a next-sibling node.
    // This next-sibling node will used as the starting point of next slice.

    // Example:
    // For a diamond inheritance relationship like this,
    //   D#1 --> B#0 --> A#0
    //     \-> C#1 -/

    // Starting point 0 stack [D]
    // Loop run #0: Stack after diving in is [D B A], A is "childless"
    // after this point, all newly visited nodes won't have a vtable that equals to a prefix of this one.
    // Loop run #0: Emitting the slice [B A] (in reverse order), B has a next-sibling node, so this slice stops here.
    // Loop run #0: Stack after exiting out is [D C], C is the next starting point.
    // Loop run #1: Stack after diving in is [D C], C is "childless", since its child A is skipped(already emitted).
    // Loop run #1: Emitting the slice [D C] (in reverse order). No one has a next-sibling node.
    // Loop run #1: Stack after exiting out is []. Now the function exits.

    loop {
        // dive deeper into the stack, recording the path
        'diving_in: loop {
            if let Some((inner_most_trait_ref, _, _)) = stack.last() {
                let inner_most_trait_ref = *inner_most_trait_ref;
                let mut direct_super_traits_iter = tcx
                    .super_predicates_of(inner_most_trait_ref.def_id())
                    .predicates
                    .into_iter()
                    .filter_map(move |(pred, _)| {
                        pred.subst_supertrait(tcx, &inner_most_trait_ref).to_opt_poly_trait_pred()
                    });

                'diving_in_skip_visited_traits: loop {
                    if let Some(next_super_trait) = direct_super_traits_iter.next() {
                        if visited.insert(next_super_trait.to_predicate(tcx)) {
                            // We're throwing away potential constness of super traits here.
                            // FIXME: handle ~const super traits
                            let next_super_trait = next_super_trait.map_bound(|t| t.trait_ref);
                            stack.push((
                                next_super_trait,
                                emit_vptr_on_new_entry,
                                Some(direct_super_traits_iter),
                            ));
                            break 'diving_in_skip_visited_traits;
                        } else {
                            continue 'diving_in_skip_visited_traits;
                        }
                    } else {
                        break 'diving_in;
                    }
                }
            }
        }

        // Other than the left-most path, vptr should be emitted for each trait.
        emit_vptr_on_new_entry = true;

        // emit innermost item, move to next sibling and stop there if possible, otherwise jump to outer level.
        'exiting_out: loop {
            if let Some((inner_most_trait_ref, emit_vptr, siblings_opt)) = stack.last_mut() {
                if let ControlFlow::Break(v) = (segment_visitor)(VtblSegment::TraitOwnEntries {
                    trait_ref: *inner_most_trait_ref,
                    emit_vptr: *emit_vptr,
                }) {
                    return Some(v);
                }

                'exiting_out_skip_visited_traits: loop {
                    if let Some(siblings) = siblings_opt {
                        if let Some(next_inner_most_trait_ref) = siblings.next() {
                            if visited.insert(next_inner_most_trait_ref.to_predicate(tcx)) {
                                // We're throwing away potential constness of super traits here.
                                // FIXME: handle ~const super traits
                                let next_inner_most_trait_ref =
                                    next_inner_most_trait_ref.map_bound(|t| t.trait_ref);
                                *inner_most_trait_ref = next_inner_most_trait_ref;
                                *emit_vptr = emit_vptr_on_new_entry;
                                break 'exiting_out;
                            } else {
                                continue 'exiting_out_skip_visited_traits;
                            }
                        }
                    }
                    stack.pop();
                    continue 'exiting_out;
                }
            }
            // all done
            return None;
        }
    }
}

fn dump_vtable_entries<'tcx>(
    tcx: TyCtxt<'tcx>,
    sp: Span,
    trait_ref: ty::PolyTraitRef<'tcx>,
    entries: &[VtblEntry<'tcx>],
) {
    tcx.sess.emit_err(DumpVTableEntries {
        span: sp,
        trait_ref,
        entries: format!("{:#?}", entries),
    });
}

fn own_existential_vtable_entries(tcx: TyCtxt<'_>, trait_def_id: DefId) -> &[DefId] {
    let trait_methods = tcx
        .associated_items(trait_def_id)
        .in_definition_order()
        .filter(|item| item.kind == ty::AssocKind::Fn);
    // Now list each method's DefId (for within its trait).
    let own_entries = trait_methods.filter_map(move |trait_method| {
        debug!("own_existential_vtable_entry: trait_method={:?}", trait_method);
        let def_id = trait_method.def_id;

        // Some methods cannot be called on an object; skip those.
        if !is_vtable_safe_method(tcx, trait_def_id, &trait_method) {
            debug!("own_existential_vtable_entry: not vtable safe");
            return None;
        }

        Some(def_id)
    });

    tcx.arena.alloc_from_iter(own_entries.into_iter())
}

/// Given a trait `trait_ref`, iterates the vtable entries
/// that come from `trait_ref`, including its supertraits.
fn vtable_entries<'tcx>(
    tcx: TyCtxt<'tcx>,
    trait_ref: ty::PolyTraitRef<'tcx>,
) -> &'tcx [VtblEntry<'tcx>] {
    debug!("vtable_entries({:?})", trait_ref);

    let mut entries = vec![];

    let vtable_segment_callback = |segment| -> ControlFlow<()> {
        match segment {
            VtblSegment::MetadataDSA => {
                entries.extend(TyCtxt::COMMON_VTABLE_ENTRIES);
            }
            VtblSegment::TraitOwnEntries { trait_ref, emit_vptr } => {
                let existential_trait_ref = trait_ref
                    .map_bound(|trait_ref| ty::ExistentialTraitRef::erase_self_ty(tcx, trait_ref));

                // Lookup the shape of vtable for the trait.
                let own_existential_entries =
                    tcx.own_existential_vtable_entries(existential_trait_ref.def_id());

                let own_entries = own_existential_entries.iter().copied().map(|def_id| {
                    debug!("vtable_entries: trait_method={:?}", def_id);

                    // The method may have some early-bound lifetimes; add regions for those.
                    let substs = trait_ref.map_bound(|trait_ref| {
                        InternalSubsts::for_item(tcx, def_id, |param, _| match param.kind {
                            GenericParamDefKind::Lifetime => tcx.lifetimes.re_erased.into(),
                            GenericParamDefKind::Type { .. }
                            | GenericParamDefKind::Const { .. } => {
                                trait_ref.substs[param.index as usize]
                            }
                        })
                    });

                    // The trait type may have higher-ranked lifetimes in it;
                    // erase them if they appear, so that we get the type
                    // at some particular call site.
                    let substs = tcx
                        .normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), substs);

                    // It's possible that the method relies on where-clauses that
                    // do not hold for this particular set of type parameters.
                    // Note that this method could then never be called, so we
                    // do not want to try and codegen it, in that case (see #23435).
                    let predicates = tcx.predicates_of(def_id).instantiate_own(tcx, substs);
                    if impossible_predicates(
                        tcx,
                        predicates.map(|(predicate, _)| predicate).collect(),
                    ) {
                        debug!("vtable_entries: predicates do not hold");
                        return VtblEntry::Vacant;
                    }

                    let instance = ty::Instance::resolve_for_vtable(
                        tcx,
                        ty::ParamEnv::reveal_all(),
                        def_id,
                        substs,
                    )
                    .expect("resolution failed during building vtable representation");
                    VtblEntry::Method(instance)
                });

                entries.extend(own_entries);

                if emit_vptr {
                    entries.push(VtblEntry::TraitVPtr(trait_ref));
                }
            }
        }

        ControlFlow::Continue(())
    };

    let _ = prepare_vtable_segments(tcx, trait_ref, vtable_segment_callback);

    if tcx.has_attr(trait_ref.def_id(), sym::rustc_dump_vtable) {
        let sp = tcx.def_span(trait_ref.def_id());
        dump_vtable_entries(tcx, sp, trait_ref, &entries);
    }

    tcx.arena.alloc_from_iter(entries.into_iter())
}

/// Find slot base for trait methods within vtable entries of another trait
pub(super) fn vtable_trait_first_method_offset<'tcx>(
    tcx: TyCtxt<'tcx>,
    key: (
        ty::PolyTraitRef<'tcx>, // trait_to_be_found
        ty::PolyTraitRef<'tcx>, // trait_owning_vtable
    ),
) -> usize {
    let (trait_to_be_found, trait_owning_vtable) = key;

    // #90177
    let trait_to_be_found_erased = tcx.erase_regions(trait_to_be_found);

    let vtable_segment_callback = {
        let mut vtable_base = 0;

        move |segment| {
            match segment {
                VtblSegment::MetadataDSA => {
                    vtable_base += TyCtxt::COMMON_VTABLE_ENTRIES.len();
                }
                VtblSegment::TraitOwnEntries { trait_ref, emit_vptr } => {
                    if tcx.erase_regions(trait_ref) == trait_to_be_found_erased {
                        return ControlFlow::Break(vtable_base);
                    }
                    vtable_base += count_own_vtable_entries(tcx, trait_ref);
                    if emit_vptr {
                        vtable_base += 1;
                    }
                }
            }
            ControlFlow::Continue(())
        }
    };

    if let Some(vtable_base) =
        prepare_vtable_segments(tcx, trait_owning_vtable, vtable_segment_callback)
    {
        vtable_base
    } else {
        bug!("Failed to find info for expected trait in vtable");
    }
}

/// Find slot offset for trait vptr within vtable entries of another trait
pub(crate) fn vtable_trait_upcasting_coercion_new_vptr_slot<'tcx>(
    tcx: TyCtxt<'tcx>,
    key: (
        Ty<'tcx>, // trait object type whose trait owning vtable
        Ty<'tcx>, // trait object for supertrait
    ),
) -> Option<usize> {
    let (source, target) = key;
    assert!(matches!(&source.kind(), &ty::Dynamic(..)) && !source.needs_infer());
    assert!(matches!(&target.kind(), &ty::Dynamic(..)) && !target.needs_infer());

    // this has been typecked-before, so diagnostics is not really needed.
    let unsize_trait_did = tcx.require_lang_item(LangItem::Unsize, None);

    let trait_ref = tcx.mk_trait_ref(unsize_trait_did, [source, target]);

    match tcx.codegen_select_candidate((ty::ParamEnv::reveal_all(), ty::Binder::dummy(trait_ref))) {
        Ok(ImplSource::TraitUpcasting(implsrc_traitcasting)) => {
            implsrc_traitcasting.vtable_vptr_slot
        }
        otherwise => bug!("expected TraitUpcasting candidate, got {otherwise:?}"),
    }
}

/// Given a trait `trait_ref`, returns the number of vtable entries
/// that come from `trait_ref`, excluding its supertraits. Used in
/// computing the vtable base for an upcast trait of a trait object.
pub(crate) fn count_own_vtable_entries<'tcx>(
    tcx: TyCtxt<'tcx>,
    trait_ref: ty::PolyTraitRef<'tcx>,
) -> usize {
    tcx.own_existential_vtable_entries(trait_ref.def_id()).len()
}

pub(super) fn provide(providers: &mut ty::query::Providers) {
    *providers = ty::query::Providers {
        own_existential_vtable_entries,
        vtable_entries,
        vtable_trait_upcasting_coercion_new_vptr_slot,
        ..*providers
    };
}
Commit	Line	Data
487cf647 FG	1	use crate::errors::DumpVTableEntries;
	2	use crate::traits::{impossible_predicates, is_vtable_safe_method};
	3	use rustc_hir::def_id::DefId;
	4	use rustc_hir::lang_items::LangItem;
	5	use rustc_infer::traits::util::PredicateSet;
	6	use rustc_infer::traits::ImplSource;
	7	use rustc_middle::ty::visit::TypeVisitable;
	8	use rustc_middle::ty::InternalSubsts;
	9	use rustc_middle::ty::{self, GenericParamDefKind, ToPredicate, Ty, TyCtxt, VtblEntry};
	10	use rustc_span::{sym, Span};
	11	use smallvec::SmallVec;
	12
	13	use std::fmt::Debug;
	14	use std::ops::ControlFlow;
	15
	16	#[derive(Clone, Debug)]
	17	pub(super) enum VtblSegment<'tcx> {
	18	MetadataDSA,
	19	TraitOwnEntries { trait_ref: ty::PolyTraitRef<'tcx>, emit_vptr: bool },
	20	}
	21
	22	/// Prepare the segments for a vtable
	23	pub(super) fn prepare_vtable_segments<'tcx, T>(
	24	tcx: TyCtxt<'tcx>,
	25	trait_ref: ty::PolyTraitRef<'tcx>,
	26	mut segment_visitor: impl FnMut(VtblSegment<'tcx>) -> ControlFlow<T>,
	27	) -> Option<T> {
	28	// The following constraints holds for the final arrangement.
	29	// 1. The whole virtual table of the first direct super trait is included as the
	30	// the prefix. If this trait doesn't have any super traits, then this step
	31	// consists of the dsa metadata.
	32	// 2. Then comes the proper pointer metadata(vptr) and all own methods for all
	33	// other super traits except those already included as part of the first
	34	// direct super trait virtual table.
	35	// 3. finally, the own methods of this trait.
	36
	37	// This has the advantage that trait upcasting to the first direct super trait on each level
	38	// is zero cost, and to another trait includes only replacing the pointer with one level indirection,
	39	// while not using too much extra memory.
	40
	41	// For a single inheritance relationship like this,
	42	// D --> C --> B --> A
	43	// The resulting vtable will consists of these segments:
	44	// DSA, A, B, C, D
	45
	46	// For a multiple inheritance relationship like this,
	47	// D --> C --> A
	48	// \-> B
	49	// The resulting vtable will consists of these segments:
	50	// DSA, A, B, B-vptr, C, D
	51
	52	// For a diamond inheritance relationship like this,
	53	// D --> B --> A
	54	// \-> C -/
	55	// The resulting vtable will consists of these segments:
	56	// DSA, A, B, C, C-vptr, D
	57
	58	// For a more complex inheritance relationship like this:
	59	// O --> G --> C --> A
	60	// \ \ \-> B
	61	// \| \|-> F --> D
	62	// \| \-> E
	63	// \|-> N --> J --> H
	64	// \ \-> I
65	// \|-> M --> K
66	// \-> L
67	// The resulting vtable will consists of these segments:
68	// DSA, A, B, B-vptr, C, D, D-vptr, E, E-vptr, F, F-vptr, G,
69	// H, H-vptr, I, I-vptr, J, J-vptr, K, K-vptr, L, L-vptr, M, M-vptr,
70	// N, N-vptr, O
71
72	// emit dsa segment first.
73	if let ControlFlow::Break(v) = (segment_visitor)(VtblSegment::MetadataDSA) {
74	return Some(v);
75	}
76
77	let mut emit_vptr_on_new_entry = false;
78	let mut visited = PredicateSet::new(tcx);
79	let predicate = trait_ref.without_const().to_predicate(tcx);
80	let mut stack: SmallVec<[(ty::PolyTraitRef<'tcx>, _, _); 5]> =
81	smallvec![(trait_ref, emit_vptr_on_new_entry, None)];
82	visited.insert(predicate);
83
84	// the main traversal loop:
85	// basically we want to cut the inheritance directed graph into a few non-overlapping slices of nodes
86	// that each node is emitted after all its descendents have been emitted.
87	// so we convert the directed graph into a tree by skipping all previously visited nodes using a visited set.
88	// this is done on the fly.
89	// Each loop run emits a slice - it starts by find a "childless" unvisited node, backtracking upwards, and it
90	// stops after it finds a node that has a next-sibling node.
91	// This next-sibling node will used as the starting point of next slice.
92
93	// Example:
94	// For a diamond inheritance relationship like this,
95	// D#1 --> B#0 --> A#0
96	// \-> C#1 -/
97
98	// Starting point 0 stack [D]
99	// Loop run #0: Stack after diving in is [D B A], A is "childless"
100	// after this point, all newly visited nodes won't have a vtable that equals to a prefix of this one.
101	// Loop run #0: Emitting the slice [B A] (in reverse order), B has a next-sibling node, so this slice stops here.
102	// Loop run #0: Stack after exiting out is [D C], C is the next starting point.
103	// Loop run #1: Stack after diving in is [D C], C is "childless", since its child A is skipped(already emitted).
104	// Loop run #1: Emitting the slice [D C] (in reverse order). No one has a next-sibling node.
105	// Loop run #1: Stack after exiting out is []. Now the function exits.
106
107	loop {
108	// dive deeper into the stack, recording the path
109	'diving_in: loop {
110	if let Some((inner_most_trait_ref, _, _)) = stack.last() {
111	let inner_most_trait_ref = *inner_most_trait_ref;
112	let mut direct_super_traits_iter = tcx
113	.super_predicates_of(inner_most_trait_ref.def_id())
114	.predicates
115	.into_iter()
116	.filter_map(move \|(pred, _)\| {
117	pred.subst_supertrait(tcx, &inner_most_trait_ref).to_opt_poly_trait_pred()
118	});
119
120	'diving_in_skip_visited_traits: loop {
121	if let Some(next_super_trait) = direct_super_traits_iter.next() {
122	if visited.insert(next_super_trait.to_predicate(tcx)) {
123	// We're throwing away potential constness of super traits here.
124	// FIXME: handle ~const super traits
125	let next_super_trait = next_super_trait.map_bound(\|t\| t.trait_ref);
126	stack.push((
127	next_super_trait,
128	emit_vptr_on_new_entry,
129	Some(direct_super_traits_iter),
130	));
131	break 'diving_in_skip_visited_traits;
132	} else {
133	continue 'diving_in_skip_visited_traits;
134	}
135	} else {
136	break 'diving_in;
137	}
138	}
139	}
140	}
141
142	// Other than the left-most path, vptr should be emitted for each trait.
143	emit_vptr_on_new_entry = true;
144
145	// emit innermost item, move to next sibling and stop there if possible, otherwise jump to outer level.
146	'exiting_out: loop {
147	if let Some((inner_most_trait_ref, emit_vptr, siblings_opt)) = stack.last_mut() {
148	if let ControlFlow::Break(v) = (segment_visitor)(VtblSegment::TraitOwnEntries {
149	trait_ref: *inner_most_trait_ref,
150	emit_vptr: *emit_vptr,
151	}) {
152	return Some(v);
153	}
154
155	'exiting_out_skip_visited_traits: loop {
156	if let Some(siblings) = siblings_opt {
157	if let Some(next_inner_most_trait_ref) = siblings.next() {
158	if visited.insert(next_inner_most_trait_ref.to_predicate(tcx)) {
159	// We're throwing away potential constness of super traits here.
160	// FIXME: handle ~const super traits
161	let next_inner_most_trait_ref =
162	next_inner_most_trait_ref.map_bound(\|t\| t.trait_ref);
163	*inner_most_trait_ref = next_inner_most_trait_ref;
164	*emit_vptr = emit_vptr_on_new_entry;
165	break 'exiting_out;
166	} else {
167	continue 'exiting_out_skip_visited_traits;
168	}
169	}
170	}
171	stack.pop();
172	continue 'exiting_out;
173	}
174	}
175	// all done
176	return None;
177	}
178	}
179	}
180
181	fn dump_vtable_entries<'tcx>(
182	tcx: TyCtxt<'tcx>,
183	sp: Span,
184	trait_ref: ty::PolyTraitRef<'tcx>,
185	entries: &[VtblEntry<'tcx>],
186	) {
187	tcx.sess.emit_err(DumpVTableEntries {
188	span: sp,
189	trait_ref,
190	entries: format!("{:#?}", entries),
191	});
192	}
193
f25598a0	194	fn own_existential_vtable_entries(tcx: TyCtxt<'_>, trait_def_id: DefId) -> &[DefId] {
487cf647 FG	195	let trait_methods = tcx
	196	.associated_items(trait_def_id)
	197	.in_definition_order()
	198	.filter(\|item\| item.kind == ty::AssocKind::Fn);
	199	// Now list each method's DefId (for within its trait).
	200	let own_entries = trait_methods.filter_map(move \|trait_method\| {
	201	debug!("own_existential_vtable_entry: trait_method={:?}", trait_method);
	202	let def_id = trait_method.def_id;
	203
	204	// Some methods cannot be called on an object; skip those.
	205	if !is_vtable_safe_method(tcx, trait_def_id, &trait_method) {
	206	debug!("own_existential_vtable_entry: not vtable safe");
	207	return None;
	208	}
	209
	210	Some(def_id)
	211	});
	212
	213	tcx.arena.alloc_from_iter(own_entries.into_iter())
	214	}
	215
	216	/// Given a trait `trait_ref`, iterates the vtable entries
	217	/// that come from `trait_ref`, including its supertraits.
	218	fn vtable_entries<'tcx>(
	219	tcx: TyCtxt<'tcx>,
	220	trait_ref: ty::PolyTraitRef<'tcx>,
	221	) -> &'tcx [VtblEntry<'tcx>] {
	222	debug!("vtable_entries({:?})", trait_ref);
	223
	224	let mut entries = vec![];
	225
	226	let vtable_segment_callback = \|segment\| -> ControlFlow<()> {
	227	match segment {
	228	VtblSegment::MetadataDSA => {
	229	entries.extend(TyCtxt::COMMON_VTABLE_ENTRIES);
	230	}
	231	VtblSegment::TraitOwnEntries { trait_ref, emit_vptr } => {
	232	let existential_trait_ref = trait_ref
	233	.map_bound(\|trait_ref\| ty::ExistentialTraitRef::erase_self_ty(tcx, trait_ref));
	234
	235	// Lookup the shape of vtable for the trait.
	236	let own_existential_entries =
	237	tcx.own_existential_vtable_entries(existential_trait_ref.def_id());
	238
	239	let own_entries = own_existential_entries.iter().copied().map(\|def_id\| {
	240	debug!("vtable_entries: trait_method={:?}", def_id);
	241
	242	// The method may have some early-bound lifetimes; add regions for those.
	243	let substs = trait_ref.map_bound(\|trait_ref\| {
	244	InternalSubsts::for_item(tcx, def_id, \|param, _\| match param.kind {
	245	GenericParamDefKind::Lifetime => tcx.lifetimes.re_erased.into(),
	246	GenericParamDefKind::Type { .. }
	247	\| GenericParamDefKind::Const { .. } => {
	248	trait_ref.substs[param.index as usize]
	249	}
	250	})
	251	});
	252
	253	// The trait type may have higher-ranked lifetimes in it;
	254	// erase them if they appear, so that we get the type
	255	// at some particular call site.
	256	let substs = tcx
	257	.normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), substs);
	258
259	// It's possible that the method relies on where-clauses that
260	// do not hold for this particular set of type parameters.
261	// Note that this method could then never be called, so we
262	// do not want to try and codegen it, in that case (see #23435).
263	let predicates = tcx.predicates_of(def_id).instantiate_own(tcx, substs);
f25598a0 FG	264	if impossible_predicates(
	265	tcx,
	266	predicates.map(\|(predicate, _)\| predicate).collect(),
	267	) {
487cf647 FG	268	debug!("vtable_entries: predicates do not hold");
	269	return VtblEntry::Vacant;
	270	}
	271
	272	let instance = ty::Instance::resolve_for_vtable(
	273	tcx,
	274	ty::ParamEnv::reveal_all(),
	275	def_id,
	276	substs,
	277	)
	278	.expect("resolution failed during building vtable representation");
	279	VtblEntry::Method(instance)
	280	});
	281
	282	entries.extend(own_entries);
	283
	284	if emit_vptr {
	285	entries.push(VtblEntry::TraitVPtr(trait_ref));
	286	}
	287	}
	288	}
	289
	290	ControlFlow::Continue(())
	291	};
	292
	293	let _ = prepare_vtable_segments(tcx, trait_ref, vtable_segment_callback);
	294
	295	if tcx.has_attr(trait_ref.def_id(), sym::rustc_dump_vtable) {
	296	let sp = tcx.def_span(trait_ref.def_id());
	297	dump_vtable_entries(tcx, sp, trait_ref, &entries);
	298	}
	299
	300	tcx.arena.alloc_from_iter(entries.into_iter())
	301	}
	302
	303	/// Find slot base for trait methods within vtable entries of another trait
	304	pub(super) fn vtable_trait_first_method_offset<'tcx>(
	305	tcx: TyCtxt<'tcx>,
	306	key: (
	307	ty::PolyTraitRef<'tcx>, // trait_to_be_found
	308	ty::PolyTraitRef<'tcx>, // trait_owning_vtable
	309	),
	310	) -> usize {
	311	let (trait_to_be_found, trait_owning_vtable) = key;
	312
	313	// #90177
	314	let trait_to_be_found_erased = tcx.erase_regions(trait_to_be_found);
	315
	316	let vtable_segment_callback = {
	317	let mut vtable_base = 0;
	318
	319	move \|segment\| {
	320	match segment {
	321	VtblSegment::MetadataDSA => {
	322	vtable_base += TyCtxt::COMMON_VTABLE_ENTRIES.len();
	323	}
	324	VtblSegment::TraitOwnEntries { trait_ref, emit_vptr } => {
	325	if tcx.erase_regions(trait_ref) == trait_to_be_found_erased {
	326	return ControlFlow::Break(vtable_base);
	327	}
	328	vtable_base += count_own_vtable_entries(tcx, trait_ref);
	329	if emit_vptr {
	330	vtable_base += 1;
	331	}
332	}
333	}
334	ControlFlow::Continue(())
335	}
336	};
337
338	if let Some(vtable_base) =
339	prepare_vtable_segments(tcx, trait_owning_vtable, vtable_segment_callback)
340	{
341	vtable_base
342	} else {
343	bug!("Failed to find info for expected trait in vtable");
344	}
345	}
346
347	/// Find slot offset for trait vptr within vtable entries of another trait
348	pub(crate) fn vtable_trait_upcasting_coercion_new_vptr_slot<'tcx>(
349	tcx: TyCtxt<'tcx>,
350	key: (
351	Ty<'tcx>, // trait object type whose trait owning vtable
352	Ty<'tcx>, // trait object for supertrait
353	),
354	) -> Option<usize> {
355	let (source, target) = key;
356	assert!(matches!(&source.kind(), &ty::Dynamic(..)) && !source.needs_infer());
357	assert!(matches!(&target.kind(), &ty::Dynamic(..)) && !target.needs_infer());
358
359	// this has been typecked-before, so diagnostics is not really needed.
360	let unsize_trait_did = tcx.require_lang_item(LangItem::Unsize, None);
361
362	let trait_ref = tcx.mk_trait_ref(unsize_trait_did, [source, target]);
363
364	match tcx.codegen_select_candidate((ty::ParamEnv::reveal_all(), ty::Binder::dummy(trait_ref))) {
365	Ok(ImplSource::TraitUpcasting(implsrc_traitcasting)) => {
366	implsrc_traitcasting.vtable_vptr_slot
367	}
368	otherwise => bug!("expected TraitUpcasting candidate, got {otherwise:?}"),
369	}
370	}
371
372	/// Given a trait `trait_ref`, returns the number of vtable entries
373	/// that come from `trait_ref`, excluding its supertraits. Used in
374	/// computing the vtable base for an upcast trait of a trait object.
375	pub(crate) fn count_own_vtable_entries<'tcx>(
376	tcx: TyCtxt<'tcx>,
377	trait_ref: ty::PolyTraitRef<'tcx>,
378	) -> usize {
379	tcx.own_existential_vtable_entries(trait_ref.def_id()).len()
380	}
381
382	pub(super) fn provide(providers: &mut ty::query::Providers) {
383	*providers = ty::query::Providers {
384	own_existential_vtable_entries,
385	vtable_entries,
386	vtable_trait_upcasting_coercion_new_vptr_slot,
387	..*providers
388	};
389	}