]>
Commit | Line | Data |
---|---|---|
dfeec247 | 1 | use crate::middle::codegen_fn_attrs::CodegenFnAttrFlags; |
dfeec247 | 2 | use crate::ty::print::{FmtPrinter, Printer}; |
9ffffee4 | 3 | use crate::ty::{self, Ty, TyCtxt, TypeFoldable, TypeSuperFoldable}; |
add651ee | 4 | use crate::ty::{EarlyBinder, GenericArgs, GenericArgsRef, TypeVisitableExt}; |
5e7ed085 | 5 | use rustc_errors::ErrorGuaranteed; |
c0240ec0 | 6 | use rustc_hir as hir; |
dfeec247 XL |
7 | use rustc_hir::def::Namespace; |
8 | use rustc_hir::def_id::{CrateNum, DefId}; | |
3dfed10e | 9 | use rustc_hir::lang_items::LangItem; |
9c376795 | 10 | use rustc_index::bit_set::FiniteBitSet; |
31ef2f64 FG |
11 | use rustc_macros::{ |
12 | Decodable, Encodable, HashStable, Lift, TyDecodable, TyEncodable, TypeVisitable, | |
13 | }; | |
a2a8927a | 14 | use rustc_middle::ty::normalize_erasing_regions::NormalizationError; |
e8be2606 | 15 | use rustc_span::def_id::LOCAL_CRATE; |
04454e1e | 16 | use rustc_span::Symbol; |
31ef2f64 | 17 | use tracing::{debug, instrument}; |
cc61c64b | 18 | |
c0240ec0 | 19 | use std::assert_matches::assert_matches; |
cc61c64b XL |
20 | use std::fmt; |
21 | ||
f035d41b XL |
22 | /// A monomorphized `InstanceDef`. |
23 | /// | |
24 | /// Monomorphization happens on-the-fly and no monomorphized MIR is ever created. Instead, this type | |
add651ee | 25 | /// simply couples a potentially generic `InstanceDef` with some args, and codegen and const eval |
c620b35d | 26 | /// will do all required instantiations as they run. |
781aab86 FG |
27 | /// |
28 | /// Note: the `Lift` impl is currently not used by rustc, but is used by | |
29 | /// rustc_codegen_cranelift when the `jit` feature is enabled. | |
3dfed10e | 30 | #[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, TyEncodable, TyDecodable)] |
f2b60f7d | 31 | #[derive(HashStable, Lift, TypeFoldable, TypeVisitable)] |
cc61c64b XL |
32 | pub struct Instance<'tcx> { |
33 | pub def: InstanceDef<'tcx>, | |
add651ee | 34 | pub args: GenericArgsRef<'tcx>, |
cc61c64b XL |
35 | } |
36 | ||
e8be2606 FG |
37 | /// Describes why a `ReifyShim` was created. This is needed to distingish a ReifyShim created to |
38 | /// adjust for things like `#[track_caller]` in a vtable from a `ReifyShim` created to produce a | |
39 | /// function pointer from a vtable entry. | |
40 | /// Currently, this is only used when KCFI is enabled, as only KCFI needs to treat those two | |
41 | /// `ReifyShim`s differently. | |
42 | #[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)] | |
43 | #[derive(TyEncodable, TyDecodable, HashStable)] | |
44 | pub enum ReifyReason { | |
45 | /// The `ReifyShim` was created to produce a function pointer. This happens when: | |
46 | /// * A vtable entry is directly converted to a function call (e.g. creating a fn ptr from a | |
47 | /// method on a `dyn` object). | |
48 | /// * A function with `#[track_caller]` is converted to a function pointer | |
49 | /// * If KCFI is enabled, creating a function pointer from a method on an object-safe trait. | |
50 | /// This includes the case of converting `::call`-like methods on closure-likes to function | |
51 | /// pointers. | |
52 | FnPtr, | |
53 | /// This `ReifyShim` was created to populate a vtable. Currently, this happens when a | |
54 | /// `#[track_caller]` mismatch occurs between the implementation of a method and the method. | |
55 | /// This includes the case of `::call`-like methods in closure-likes' vtables. | |
56 | Vtable, | |
57 | } | |
58 | ||
c620b35d | 59 | #[derive(Copy, Clone, PartialEq, Eq, Hash, Debug)] |
f2b60f7d | 60 | #[derive(TyEncodable, TyDecodable, HashStable, TypeFoldable, TypeVisitable, Lift)] |
cc61c64b | 61 | pub enum InstanceDef<'tcx> { |
f035d41b XL |
62 | /// A user-defined callable item. |
63 | /// | |
64 | /// This includes: | |
65 | /// - `fn` items | |
66 | /// - closures | |
ed00b5ec | 67 | /// - coroutines |
49aad941 | 68 | Item(DefId), |
f035d41b XL |
69 | |
70 | /// An intrinsic `fn` item (with `"rust-intrinsic"` or `"platform-intrinsic"` ABI). | |
71 | /// | |
72 | /// Alongside `Virtual`, this is the only `InstanceDef` that does not have its own callable MIR. | |
73 | /// Instead, codegen and const eval "magically" evaluate calls to intrinsics purely in the | |
74 | /// caller. | |
cc61c64b | 75 | Intrinsic(DefId), |
3b2f2976 | 76 | |
f035d41b XL |
77 | /// `<T as Trait>::method` where `method` receives unsizeable `self: Self` (part of the |
78 | /// `unsized_locals` feature). | |
79 | /// | |
80 | /// The generated shim will take `Self` via `*mut Self` - conceptually this is `&owned Self` - | |
81 | /// and dereference the argument to call the original function. | |
064997fb | 82 | VTableShim(DefId), |
a1dfa0c6 | 83 | |
e74abb32 XL |
84 | /// `fn()` pointer where the function itself cannot be turned into a pointer. |
85 | /// | |
60c5eb7d XL |
86 | /// One example is `<dyn Trait as Trait>::fn`, where the shim contains |
87 | /// a virtual call, which codegen supports only via a direct call to the | |
88 | /// `<dyn Trait as Trait>::fn` instance (an `InstanceDef::Virtual`). | |
89 | /// | |
90 | /// Another example is functions annotated with `#[track_caller]`, which | |
91 | /// must have their implicit caller location argument populated for a call. | |
92 | /// Because this is a required part of the function's ABI but can't be tracked | |
93 | /// as a property of the function pointer, we use a single "caller location" | |
94 | /// (the definition of the function itself). | |
e8be2606 FG |
95 | /// |
96 | /// The second field encodes *why* this shim was created. This allows distinguishing between | |
97 | /// a `ReifyShim` that appears in a vtable vs one that appears as a function pointer. | |
98 | /// | |
99 | /// This field will only be populated if we are compiling in a mode that needs these shims | |
100 | /// to be separable, currently only when KCFI is enabled. | |
101 | ReifyShim(DefId, Option<ReifyReason>), | |
e74abb32 | 102 | |
f035d41b XL |
103 | /// `<fn() as FnTrait>::call_*` (generated `FnTrait` implementation for `fn()` pointers). |
104 | /// | |
60c5eb7d | 105 | /// `DefId` is `FnTrait::call_*`. |
cc61c64b | 106 | FnPtrShim(DefId, Ty<'tcx>), |
3b2f2976 | 107 | |
f035d41b | 108 | /// Dynamic dispatch to `<dyn Trait as Trait>::fn`. |
60c5eb7d | 109 | /// |
f035d41b XL |
110 | /// This `InstanceDef` does not have callable MIR. Calls to `Virtual` instances must be |
111 | /// codegen'd as virtual calls through the vtable. | |
112 | /// | |
113 | /// If this is reified to a `fn` pointer, a `ReifyShim` is used (see `ReifyShim` above for more | |
114 | /// details on that). | |
cc61c64b | 115 | Virtual(DefId, usize), |
3b2f2976 | 116 | |
c620b35d | 117 | /// `<[FnMut/Fn closure] as FnOnce>::call_once`. |
f035d41b XL |
118 | /// |
119 | /// The `DefId` is the ID of the `call_once` method in `FnOnce`. | |
c620b35d FG |
120 | /// |
121 | /// This generates a body that will just borrow the (owned) self type, | |
122 | /// and dispatch to the `FnMut::call_mut` instance for the closure. | |
c295e0f8 | 123 | ClosureOnceShim { call_once: DefId, track_caller: bool }, |
3b2f2976 | 124 | |
e8be2606 | 125 | /// `<[FnMut/Fn coroutine-closure] as FnOnce>::call_once` |
c620b35d FG |
126 | /// |
127 | /// The body generated here differs significantly from the `ClosureOnceShim`, | |
128 | /// since we need to generate a distinct coroutine type that will move the | |
129 | /// closure's upvars *out* of the closure. | |
130 | ConstructCoroutineInClosureShim { | |
131 | coroutine_closure_def_id: DefId, | |
e8be2606 FG |
132 | // Whether the generated MIR body takes the coroutine by-ref. This is |
133 | // because the signature of `<{async fn} as FnMut>::call_mut` is: | |
134 | // `fn(&mut self, args: A) -> <Self as FnOnce>::Output`, that is to say | |
135 | // that it returns the `FnOnce`-flavored coroutine but takes the closure | |
136 | // by mut ref (and similarly for `Fn::call`). | |
137 | receiver_by_ref: bool, | |
c620b35d FG |
138 | }, |
139 | ||
140 | /// `<[coroutine] as Future>::poll`, but for coroutines produced when `AsyncFnOnce` | |
141 | /// is called on a coroutine-closure whose closure kind greater than `FnOnce`, or | |
142 | /// similarly for `AsyncFnMut`. | |
143 | /// | |
144 | /// This will select the body that is produced by the `ByMoveBody` transform, and thus | |
145 | /// take and use all of its upvars by-move rather than by-ref. | |
e8be2606 | 146 | CoroutineKindShim { coroutine_def_id: DefId }, |
c620b35d | 147 | |
353b0b11 FG |
148 | /// Compiler-generated accessor for thread locals which returns a reference to the thread local |
149 | /// the `DefId` defines. This is used to export thread locals from dylibs on platforms lacking | |
150 | /// native support. | |
151 | ThreadLocalShim(DefId), | |
152 | ||
74b04a01 | 153 | /// `core::ptr::drop_in_place::<T>`. |
f035d41b | 154 | /// |
74b04a01 XL |
155 | /// The `DefId` is for `core::ptr::drop_in_place`. |
156 | /// The `Option<Ty<'tcx>>` is either `Some(T)`, or `None` for empty drop | |
157 | /// glue. | |
cc61c64b | 158 | DropGlue(DefId, Option<Ty<'tcx>>), |
3b2f2976 | 159 | |
f035d41b XL |
160 | /// Compiler-generated `<T as Clone>::clone` implementation. |
161 | /// | |
162 | /// For all types that automatically implement `Copy`, a trivial `Clone` impl is provided too. | |
163 | /// Additionally, arrays, tuples, and closures get a `Clone` shim even if they aren't `Copy`. | |
164 | /// | |
165 | /// The `DefId` is for `Clone::clone`, the `Ty` is the type `T` with the builtin `Clone` impl. | |
3b2f2976 | 166 | CloneShim(DefId, Ty<'tcx>), |
353b0b11 FG |
167 | |
168 | /// Compiler-generated `<T as FnPtr>::addr` implementation. | |
169 | /// | |
170 | /// Automatically generated for all potentially higher-ranked `fn(I) -> R` types. | |
171 | /// | |
172 | /// The `DefId` is for `FnPtr::addr`, the `Ty` is the type `T`. | |
173 | FnPtrAddrShim(DefId, Ty<'tcx>), | |
e8be2606 FG |
174 | |
175 | /// `core::future::async_drop::async_drop_in_place::<'_, T>`. | |
176 | /// | |
177 | /// The `DefId` is for `core::future::async_drop::async_drop_in_place`, the `Ty` | |
178 | /// is the type `T`. | |
179 | AsyncDropGlueCtorShim(DefId, Option<Ty<'tcx>>), | |
cc61c64b XL |
180 | } |
181 | ||
dc9dc135 | 182 | impl<'tcx> Instance<'tcx> { |
c620b35d | 183 | /// Returns the `Ty` corresponding to this `Instance`, with generic instantiations applied and |
3dfed10e XL |
184 | /// lifetimes erased, allowing a `ParamEnv` to be specified for use during normalization. |
185 | pub fn ty(&self, tcx: TyCtxt<'tcx>, param_env: ty::ParamEnv<'tcx>) -> Ty<'tcx> { | |
ff7c6d11 | 186 | let ty = tcx.type_of(self.def.def_id()); |
781aab86 | 187 | tcx.instantiate_and_normalize_erasing_regions(self.args, param_env, ty) |
dfeec247 XL |
188 | } |
189 | ||
190 | /// Finds a crate that contains a monomorphization of this instance that | |
191 | /// can be linked to from the local crate. A return value of `None` means | |
192 | /// no upstream crate provides such an exported monomorphization. | |
193 | /// | |
194 | /// This method already takes into account the global `-Zshare-generics` | |
195 | /// setting, always returning `None` if `share-generics` is off. | |
196 | pub fn upstream_monomorphization(&self, tcx: TyCtxt<'tcx>) -> Option<CrateNum> { | |
197 | // If we are not in share generics mode, we don't link to upstream | |
198 | // monomorphizations but always instantiate our own internal versions | |
199 | // instead. | |
200 | if !tcx.sess.opts.share_generics() { | |
201 | return None; | |
202 | } | |
203 | ||
204 | // If this is an item that is defined in the local crate, no upstream | |
205 | // crate can know about it/provide a monomorphization. | |
206 | if self.def_id().is_local() { | |
207 | return None; | |
208 | } | |
209 | ||
210 | // If this a non-generic instance, it cannot be a shared monomorphization. | |
781aab86 | 211 | self.args.non_erasable_generics(tcx, self.def_id()).next()?; |
dfeec247 | 212 | |
e8be2606 FG |
213 | // compiler_builtins cannot use upstream monomorphizations. |
214 | if tcx.is_compiler_builtins(LOCAL_CRATE) { | |
215 | return None; | |
216 | } | |
217 | ||
dfeec247 | 218 | match self.def { |
3dfed10e | 219 | InstanceDef::Item(def) => tcx |
49aad941 | 220 | .upstream_monomorphizations_for(def) |
add651ee | 221 | .and_then(|monos| monos.get(&self.args).cloned()), |
e8be2606 FG |
222 | InstanceDef::DropGlue(_, Some(_)) | InstanceDef::AsyncDropGlueCtorShim(_, _) => { |
223 | tcx.upstream_drop_glue_for(self.args) | |
224 | } | |
dfeec247 XL |
225 | _ => None, |
226 | } | |
ff7c6d11 XL |
227 | } |
228 | } | |
229 | ||
cc61c64b XL |
230 | impl<'tcx> InstanceDef<'tcx> { |
231 | #[inline] | |
3dfed10e XL |
232 | pub fn def_id(self) -> DefId { |
233 | match self { | |
49aad941 FG |
234 | InstanceDef::Item(def_id) |
235 | | InstanceDef::VTableShim(def_id) | |
e8be2606 | 236 | | InstanceDef::ReifyShim(def_id, _) |
dfeec247 XL |
237 | | InstanceDef::FnPtrShim(def_id, _) |
238 | | InstanceDef::Virtual(def_id, _) | |
239 | | InstanceDef::Intrinsic(def_id) | |
353b0b11 | 240 | | InstanceDef::ThreadLocalShim(def_id) |
c295e0f8 | 241 | | InstanceDef::ClosureOnceShim { call_once: def_id, track_caller: _ } |
c620b35d FG |
242 | | ty::InstanceDef::ConstructCoroutineInClosureShim { |
243 | coroutine_closure_def_id: def_id, | |
e8be2606 | 244 | receiver_by_ref: _, |
c620b35d | 245 | } |
e8be2606 | 246 | | ty::InstanceDef::CoroutineKindShim { coroutine_def_id: def_id } |
dfeec247 | 247 | | InstanceDef::DropGlue(def_id, _) |
353b0b11 | 248 | | InstanceDef::CloneShim(def_id, _) |
e8be2606 FG |
249 | | InstanceDef::FnPtrAddrShim(def_id, _) |
250 | | InstanceDef::AsyncDropGlueCtorShim(def_id, _) => def_id, | |
cc61c64b XL |
251 | } |
252 | } | |
253 | ||
c295e0f8 XL |
254 | /// Returns the `DefId` of instances which might not require codegen locally. |
255 | pub fn def_id_if_not_guaranteed_local_codegen(self) -> Option<DefId> { | |
256 | match self { | |
49aad941 | 257 | ty::InstanceDef::Item(def) => Some(def), |
e8be2606 FG |
258 | ty::InstanceDef::DropGlue(def_id, Some(_)) |
259 | | InstanceDef::AsyncDropGlueCtorShim(def_id, _) | |
260 | | InstanceDef::ThreadLocalShim(def_id) => Some(def_id), | |
064997fb | 261 | InstanceDef::VTableShim(..) |
c295e0f8 XL |
262 | | InstanceDef::ReifyShim(..) |
263 | | InstanceDef::FnPtrShim(..) | |
264 | | InstanceDef::Virtual(..) | |
265 | | InstanceDef::Intrinsic(..) | |
266 | | InstanceDef::ClosureOnceShim { .. } | |
c620b35d FG |
267 | | ty::InstanceDef::ConstructCoroutineInClosureShim { .. } |
268 | | ty::InstanceDef::CoroutineKindShim { .. } | |
c295e0f8 | 269 | | InstanceDef::DropGlue(..) |
353b0b11 FG |
270 | | InstanceDef::CloneShim(..) |
271 | | InstanceDef::FnPtrAddrShim(..) => None, | |
c295e0f8 XL |
272 | } |
273 | } | |
274 | ||
cc61c64b | 275 | #[inline] |
353b0b11 FG |
276 | pub fn get_attrs( |
277 | &self, | |
278 | tcx: TyCtxt<'tcx>, | |
279 | attr: Symbol, | |
280 | ) -> impl Iterator<Item = &'tcx rustc_ast::Attribute> { | |
04454e1e | 281 | tcx.get_attrs(self.def_id(), attr) |
cc61c64b | 282 | } |
ff7c6d11 | 283 | |
dfeec247 XL |
284 | /// Returns `true` if the LLVM version of this instance is unconditionally |
285 | /// marked with `inline`. This implies that a copy of this instance is | |
286 | /// generated in every codegen unit. | |
287 | /// Note that this is only a hint. See the documentation for | |
288 | /// `generates_cgu_internal_copy` for more information. | |
289 | pub fn requires_inline(&self, tcx: TyCtxt<'tcx>) -> bool { | |
ba9703b0 | 290 | use rustc_hir::definitions::DefPathData; |
ff7c6d11 | 291 | let def_id = match *self { |
49aad941 | 292 | ty::InstanceDef::Item(def) => def, |
ff7c6d11 | 293 | ty::InstanceDef::DropGlue(_, Some(_)) => return false, |
e8be2606 | 294 | ty::InstanceDef::AsyncDropGlueCtorShim(_, Some(_)) => return false, |
353b0b11 | 295 | ty::InstanceDef::ThreadLocalShim(_) => return false, |
dfeec247 | 296 | _ => return true, |
ff7c6d11 | 297 | }; |
29967ef6 XL |
298 | matches!( |
299 | tcx.def_key(def_id).disambiguated_data.data, | |
4b012472 | 300 | DefPathData::Ctor | DefPathData::Closure |
29967ef6 | 301 | ) |
ff7c6d11 XL |
302 | } |
303 | ||
dfeec247 XL |
304 | /// Returns `true` if the machine code for this instance is instantiated in |
305 | /// each codegen unit that references it. | |
306 | /// Note that this is only a hint! The compiler can globally decide to *not* | |
307 | /// do this in order to speed up compilation. CGU-internal copies are | |
308 | /// only exist to enable inlining. If inlining is not performed (e.g. at | |
309 | /// `-Copt-level=0`) then the time for generating them is wasted and it's | |
310 | /// better to create a single copy with external linkage. | |
311 | pub fn generates_cgu_internal_copy(&self, tcx: TyCtxt<'tcx>) -> bool { | |
312 | if self.requires_inline(tcx) { | |
313 | return true; | |
ff7c6d11 | 314 | } |
74b04a01 XL |
315 | if let ty::InstanceDef::DropGlue(.., Some(ty)) = *self { |
316 | // Drop glue generally wants to be instantiated at every codegen | |
ff7c6d11 XL |
317 | // unit, but without an #[inline] hint. We should make this |
318 | // available to normal end-users. | |
74b04a01 XL |
319 | if tcx.sess.opts.incremental.is_none() { |
320 | return true; | |
321 | } | |
322 | // When compiling with incremental, we can generate a *lot* of | |
323 | // codegen units. Including drop glue into all of them has a | |
324 | // considerable compile time cost. | |
325 | // | |
326 | // We include enums without destructors to allow, say, optimizing | |
327 | // drops of `Option::None` before LTO. We also respect the intent of | |
328 | // `#[inline]` on `Drop::drop` implementations. | |
329 | return ty.ty_adt_def().map_or(true, |adt_def| { | |
ed00b5ec FG |
330 | adt_def |
331 | .destructor(tcx) | |
332 | .map_or_else(|| adt_def.is_enum(), |dtor| tcx.cross_crate_inlinable(dtor.did)) | |
74b04a01 | 333 | }); |
ff7c6d11 | 334 | } |
353b0b11 FG |
335 | if let ty::InstanceDef::ThreadLocalShim(..) = *self { |
336 | return false; | |
337 | } | |
ed00b5ec | 338 | tcx.cross_crate_inlinable(self.def_id()) |
ff7c6d11 | 339 | } |
60c5eb7d XL |
340 | |
341 | pub fn requires_caller_location(&self, tcx: TyCtxt<'_>) -> bool { | |
dfeec247 | 342 | match *self { |
49aad941 | 343 | InstanceDef::Item(def_id) | InstanceDef::Virtual(def_id, _) => { |
04454e1e | 344 | tcx.body_codegen_attrs(def_id).flags.contains(CodegenFnAttrFlags::TRACK_CALLER) |
dfeec247 | 345 | } |
c295e0f8 | 346 | InstanceDef::ClosureOnceShim { call_once: _, track_caller } => track_caller, |
dfeec247 XL |
347 | _ => false, |
348 | } | |
60c5eb7d | 349 | } |
1b1a35ee XL |
350 | |
351 | /// Returns `true` when the MIR body associated with this instance should be monomorphized | |
c620b35d | 352 | /// by its users (e.g. codegen or miri) by instantiating the `args` from `Instance` (see |
add651ee | 353 | /// `Instance::args_for_mir_body`). |
1b1a35ee XL |
354 | /// |
355 | /// Otherwise, returns `false` only for some kinds of shims where the construction of the MIR | |
c620b35d | 356 | /// body should perform necessary instantiations. |
1b1a35ee XL |
357 | pub fn has_polymorphic_mir_body(&self) -> bool { |
358 | match *self { | |
359 | InstanceDef::CloneShim(..) | |
353b0b11 FG |
360 | | InstanceDef::ThreadLocalShim(..) |
361 | | InstanceDef::FnPtrAddrShim(..) | |
1b1a35ee | 362 | | InstanceDef::FnPtrShim(..) |
e8be2606 FG |
363 | | InstanceDef::DropGlue(_, Some(_)) |
364 | | InstanceDef::AsyncDropGlueCtorShim(_, Some(_)) => false, | |
1b1a35ee | 365 | InstanceDef::ClosureOnceShim { .. } |
c620b35d FG |
366 | | InstanceDef::ConstructCoroutineInClosureShim { .. } |
367 | | InstanceDef::CoroutineKindShim { .. } | |
1b1a35ee | 368 | | InstanceDef::DropGlue(..) |
e8be2606 | 369 | | InstanceDef::AsyncDropGlueCtorShim(..) |
1b1a35ee XL |
370 | | InstanceDef::Item(_) |
371 | | InstanceDef::Intrinsic(..) | |
372 | | InstanceDef::ReifyShim(..) | |
373 | | InstanceDef::Virtual(..) | |
064997fb | 374 | | InstanceDef::VTableShim(..) => true, |
1b1a35ee XL |
375 | } |
376 | } | |
cc61c64b XL |
377 | } |
378 | ||
487cf647 FG |
379 | fn fmt_instance( |
380 | f: &mut fmt::Formatter<'_>, | |
e8be2606 | 381 | instance: Instance<'_>, |
c0240ec0 | 382 | type_length: Option<rustc_session::Limit>, |
487cf647 FG |
383 | ) -> fmt::Result { |
384 | ty::tls::with(|tcx| { | |
add651ee | 385 | let args = tcx.lift(instance.args).expect("could not lift for printing"); |
487cf647 | 386 | |
c0240ec0 FG |
387 | let mut cx = if let Some(type_length) = type_length { |
388 | FmtPrinter::new_with_limit(tcx, Namespace::ValueNS, type_length) | |
389 | } else { | |
390 | FmtPrinter::new(tcx, Namespace::ValueNS) | |
391 | }; | |
ed00b5ec FG |
392 | cx.print_def_path(instance.def_id(), args)?; |
393 | let s = cx.into_buffer(); | |
487cf647 FG |
394 | f.write_str(&s) |
395 | })?; | |
396 | ||
397 | match instance.def { | |
398 | InstanceDef::Item(_) => Ok(()), | |
399 | InstanceDef::VTableShim(_) => write!(f, " - shim(vtable)"), | |
e8be2606 FG |
400 | InstanceDef::ReifyShim(_, None) => write!(f, " - shim(reify)"), |
401 | InstanceDef::ReifyShim(_, Some(ReifyReason::FnPtr)) => write!(f, " - shim(reify-fnptr)"), | |
402 | InstanceDef::ReifyShim(_, Some(ReifyReason::Vtable)) => write!(f, " - shim(reify-vtable)"), | |
353b0b11 | 403 | InstanceDef::ThreadLocalShim(_) => write!(f, " - shim(tls)"), |
487cf647 | 404 | InstanceDef::Intrinsic(_) => write!(f, " - intrinsic"), |
add651ee FG |
405 | InstanceDef::Virtual(_, num) => write!(f, " - virtual#{num}"), |
406 | InstanceDef::FnPtrShim(_, ty) => write!(f, " - shim({ty})"), | |
487cf647 | 407 | InstanceDef::ClosureOnceShim { .. } => write!(f, " - shim"), |
c620b35d FG |
408 | InstanceDef::ConstructCoroutineInClosureShim { .. } => write!(f, " - shim"), |
409 | InstanceDef::CoroutineKindShim { .. } => write!(f, " - shim"), | |
487cf647 | 410 | InstanceDef::DropGlue(_, None) => write!(f, " - shim(None)"), |
add651ee FG |
411 | InstanceDef::DropGlue(_, Some(ty)) => write!(f, " - shim(Some({ty}))"), |
412 | InstanceDef::CloneShim(_, ty) => write!(f, " - shim({ty})"), | |
413 | InstanceDef::FnPtrAddrShim(_, ty) => write!(f, " - shim({ty})"), | |
e8be2606 FG |
414 | InstanceDef::AsyncDropGlueCtorShim(_, None) => write!(f, " - shim(None)"), |
415 | InstanceDef::AsyncDropGlueCtorShim(_, Some(ty)) => write!(f, " - shim(Some({ty}))"), | |
487cf647 FG |
416 | } |
417 | } | |
418 | ||
e8be2606 | 419 | pub struct ShortInstance<'tcx>(pub Instance<'tcx>, pub usize); |
487cf647 | 420 | |
e8be2606 | 421 | impl<'tcx> fmt::Display for ShortInstance<'tcx> { |
0bf4aa26 | 422 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { |
c0240ec0 | 423 | fmt_instance(f, self.0, Some(rustc_session::Limit(self.1))) |
487cf647 FG |
424 | } |
425 | } | |
532ac7d7 | 426 | |
487cf647 FG |
427 | impl<'tcx> fmt::Display for Instance<'tcx> { |
428 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { | |
e8be2606 | 429 | fmt_instance(f, *self, None) |
cc61c64b XL |
430 | } |
431 | } | |
432 | ||
dc9dc135 | 433 | impl<'tcx> Instance<'tcx> { |
add651ee | 434 | pub fn new(def_id: DefId, args: GenericArgsRef<'tcx>) -> Instance<'tcx> { |
dfeec247 | 435 | assert!( |
add651ee FG |
436 | !args.has_escaping_bound_vars(), |
437 | "args of instance {def_id:?} not normalized for codegen: {args:?}" | |
dfeec247 | 438 | ); |
add651ee | 439 | Instance { def: InstanceDef::Item(def_id), args } |
cc61c64b XL |
440 | } |
441 | ||
dc9dc135 | 442 | pub fn mono(tcx: TyCtxt<'tcx>, def_id: DefId) -> Instance<'tcx> { |
add651ee | 443 | let args = GenericArgs::for_item(tcx, def_id, |param, _| match param.kind { |
29967ef6 | 444 | ty::GenericParamDefKind::Lifetime => tcx.lifetimes.re_erased.into(), |
781aab86 | 445 | ty::GenericParamDefKind::Const { is_host_effect: true, .. } => tcx.consts.true_.into(), |
29967ef6 XL |
446 | ty::GenericParamDefKind::Type { .. } => { |
447 | bug!("Instance::mono: {:?} has type parameters", def_id) | |
448 | } | |
449 | ty::GenericParamDefKind::Const { .. } => { | |
450 | bug!("Instance::mono: {:?} has const parameters", def_id) | |
451 | } | |
452 | }); | |
453 | ||
add651ee | 454 | Instance::new(def_id, args) |
cc61c64b XL |
455 | } |
456 | ||
457 | #[inline] | |
458 | pub fn def_id(&self) -> DefId { | |
459 | self.def.def_id() | |
460 | } | |
ea8adc8c | 461 | |
add651ee | 462 | /// Resolves a `(def_id, args)` pair to an (optional) instance -- most commonly, |
ea8adc8c XL |
463 | /// this is used to find the precise code that will run for a trait method invocation, |
464 | /// if known. | |
465 | /// | |
f9f354fc | 466 | /// Returns `Ok(None)` if we cannot resolve `Instance` to a specific instance. |
ea8adc8c XL |
467 | /// For example, in a context like this, |
468 | /// | |
04454e1e | 469 | /// ```ignore (illustrative) |
ea8adc8c XL |
470 | /// fn foo<T: Debug>(t: T) { ... } |
471 | /// ``` | |
472 | /// | |
f9f354fc | 473 | /// trying to resolve `Debug::fmt` applied to `T` will yield `Ok(None)`, because we do not |
ea8adc8c XL |
474 | /// know what code ought to run. (Note that this setting is also affected by the |
475 | /// `RevealMode` in the parameter environment.) | |
476 | /// | |
477 | /// Presuming that coherence and type-check have succeeded, if this method is invoked | |
94b46f34 | 478 | /// in a monomorphic context (i.e., like during codegen), then it is guaranteed to return |
f9f354fc XL |
479 | /// `Ok(Some(instance))`. |
480 | /// | |
5e7ed085 | 481 | /// Returns `Err(ErrorGuaranteed)` when the `Instance` resolution process |
f9f354fc XL |
482 | /// couldn't complete due to errors elsewhere - this is distinct |
483 | /// from `Ok(None)` to avoid misleading diagnostics when an error | |
484 | /// has already been/will be emitted, for the original cause | |
49aad941 | 485 | #[instrument(level = "debug", skip(tcx), ret)] |
dc9dc135 XL |
486 | pub fn resolve( |
487 | tcx: TyCtxt<'tcx>, | |
488 | param_env: ty::ParamEnv<'tcx>, | |
489 | def_id: DefId, | |
add651ee | 490 | args: GenericArgsRef<'tcx>, |
5e7ed085 | 491 | ) -> Result<Option<Instance<'tcx>>, ErrorGuaranteed> { |
49aad941 FG |
492 | // All regions in the result of this query are erased, so it's |
493 | // fine to erase all of the input regions. | |
494 | ||
add651ee | 495 | // HACK(eddyb) erase regions in `args` first, so that `param_env.and(...)` |
49aad941 | 496 | // below is more likely to ignore the bounds in scope (e.g. if the only |
add651ee FG |
497 | // generic parameters mentioned by `args` were lifetime ones). |
498 | let args = tcx.erase_regions(args); | |
499 | tcx.resolve_instance(tcx.erase_regions(param_env.and((def_id, args)))) | |
3dfed10e XL |
500 | } |
501 | ||
9c376795 FG |
502 | pub fn expect_resolve( |
503 | tcx: TyCtxt<'tcx>, | |
504 | param_env: ty::ParamEnv<'tcx>, | |
505 | def_id: DefId, | |
add651ee | 506 | args: GenericArgsRef<'tcx>, |
9c376795 | 507 | ) -> Instance<'tcx> { |
add651ee | 508 | match ty::Instance::resolve(tcx, param_env, def_id, args) { |
9c376795 | 509 | Ok(Some(instance)) => instance, |
fe692bf9 FG |
510 | instance => bug!( |
511 | "failed to resolve instance for {}: {instance:#?}", | |
add651ee | 512 | tcx.def_path_str_with_args(def_id, args) |
9c376795 FG |
513 | ), |
514 | } | |
515 | } | |
516 | ||
e74abb32 XL |
517 | pub fn resolve_for_fn_ptr( |
518 | tcx: TyCtxt<'tcx>, | |
519 | param_env: ty::ParamEnv<'tcx>, | |
520 | def_id: DefId, | |
add651ee | 521 | args: GenericArgsRef<'tcx>, |
e74abb32 | 522 | ) -> Option<Instance<'tcx>> { |
add651ee | 523 | debug!("resolve(def_id={:?}, args={:?})", def_id, args); |
c295e0f8 | 524 | // Use either `resolve_closure` or `resolve_for_vtable` |
c620b35d | 525 | assert!(!tcx.is_closure_like(def_id), "Called `resolve_for_fn_ptr` on closure: {def_id:?}"); |
e8be2606 | 526 | let reason = tcx.sess.is_sanitizer_kcfi_enabled().then_some(ReifyReason::FnPtr); |
add651ee | 527 | Instance::resolve(tcx, param_env, def_id, args).ok().flatten().map(|mut resolved| { |
e74abb32 | 528 | match resolved.def { |
3dfed10e | 529 | InstanceDef::Item(def) if resolved.def.requires_caller_location(tcx) => { |
e74abb32 | 530 | debug!(" => fn pointer created for function with #[track_caller]"); |
e8be2606 | 531 | resolved.def = InstanceDef::ReifyShim(def, reason); |
60c5eb7d XL |
532 | } |
533 | InstanceDef::Virtual(def_id, _) => { | |
534 | debug!(" => fn pointer created for virtual call"); | |
e8be2606 FG |
535 | resolved.def = InstanceDef::ReifyShim(def_id, reason); |
536 | } | |
537 | // Reify `Trait::method` implementations if KCFI is enabled | |
538 | // FIXME(maurer) only reify it if it is a vtable-safe function | |
539 | _ if tcx.sess.is_sanitizer_kcfi_enabled() | |
540 | && tcx | |
541 | .opt_associated_item(def_id) | |
542 | .and_then(|assoc| assoc.trait_item_def_id) | |
543 | .is_some() => | |
544 | { | |
545 | // If this function could also go in a vtable, we need to `ReifyShim` it with | |
546 | // KCFI because it can only attach one type per function. | |
547 | resolved.def = InstanceDef::ReifyShim(resolved.def_id(), reason) | |
548 | } | |
549 | // Reify `::call`-like method implementations if KCFI is enabled | |
550 | _ if tcx.sess.is_sanitizer_kcfi_enabled() | |
551 | && tcx.is_closure_like(resolved.def_id()) => | |
552 | { | |
553 | // Reroute through a reify via the *unresolved* instance. The resolved one can't | |
554 | // be directly reified because it's closure-like. The reify can handle the | |
555 | // unresolved instance. | |
556 | resolved = Instance { def: InstanceDef::ReifyShim(def_id, reason), args } | |
60c5eb7d XL |
557 | } |
558 | _ => {} | |
e74abb32 | 559 | } |
60c5eb7d XL |
560 | |
561 | resolved | |
e74abb32 XL |
562 | }) |
563 | } | |
564 | ||
dc9dc135 XL |
565 | pub fn resolve_for_vtable( |
566 | tcx: TyCtxt<'tcx>, | |
567 | param_env: ty::ParamEnv<'tcx>, | |
568 | def_id: DefId, | |
add651ee | 569 | args: GenericArgsRef<'tcx>, |
dc9dc135 | 570 | ) -> Option<Instance<'tcx>> { |
add651ee FG |
571 | debug!("resolve_for_vtable(def_id={:?}, args={:?})", def_id, args); |
572 | let fn_sig = tcx.fn_sig(def_id).instantiate_identity(); | |
74b04a01 | 573 | let is_vtable_shim = !fn_sig.inputs().skip_binder().is_empty() |
e1599b0c XL |
574 | && fn_sig.input(0).skip_binder().is_param(0) |
575 | && tcx.generics_of(def_id).has_self; | |
a1dfa0c6 XL |
576 | if is_vtable_shim { |
577 | debug!(" => associated item with unsizeable self: Self"); | |
add651ee | 578 | Some(Instance { def: InstanceDef::VTableShim(def_id), args }) |
a1dfa0c6 | 579 | } else { |
e8be2606 | 580 | let reason = tcx.sess.is_sanitizer_kcfi_enabled().then_some(ReifyReason::Vtable); |
add651ee | 581 | Instance::resolve(tcx, param_env, def_id, args).ok().flatten().map(|mut resolved| { |
136023e0 XL |
582 | match resolved.def { |
583 | InstanceDef::Item(def) => { | |
584 | // We need to generate a shim when we cannot guarantee that | |
585 | // the caller of a trait object method will be aware of | |
586 | // `#[track_caller]` - this ensures that the caller | |
587 | // and callee ABI will always match. | |
588 | // | |
589 | // The shim is generated when all of these conditions are met: | |
590 | // | |
591 | // 1) The underlying method expects a caller location parameter | |
592 | // in the ABI | |
593 | if resolved.def.requires_caller_location(tcx) | |
594 | // 2) The caller location parameter comes from having `#[track_caller]` | |
595 | // on the implementation, and *not* on the trait method. | |
49aad941 | 596 | && !tcx.should_inherit_track_caller(def) |
136023e0 XL |
597 | // If the method implementation comes from the trait definition itself |
598 | // (e.g. `trait Foo { #[track_caller] my_fn() { /* impl */ } }`), | |
599 | // then we don't need to generate a shim. This check is needed because | |
600 | // `should_inherit_track_caller` returns `false` if our method | |
601 | // implementation comes from the trait block, and not an impl block | |
602 | && !matches!( | |
49aad941 | 603 | tcx.opt_associated_item(def), |
136023e0 | 604 | Some(ty::AssocItem { |
064997fb | 605 | container: ty::AssocItemContainer::TraitContainer, |
136023e0 XL |
606 | .. |
607 | }) | |
608 | ) | |
609 | { | |
c620b35d | 610 | if tcx.is_closure_like(def) { |
c295e0f8 | 611 | debug!(" => vtable fn pointer created for closure with #[track_caller]: {:?} for method {:?} {:?}", |
add651ee | 612 | def, def_id, args); |
c295e0f8 XL |
613 | |
614 | // Create a shim for the `FnOnce/FnMut/Fn` method we are calling | |
615 | // - unlike functions, invoking a closure always goes through a | |
616 | // trait. | |
e8be2606 | 617 | resolved = Instance { def: InstanceDef::ReifyShim(def_id, reason), args }; |
c295e0f8 XL |
618 | } else { |
619 | debug!( | |
49aad941 | 620 | " => vtable fn pointer created for function with #[track_caller]: {:?}", def |
c295e0f8 | 621 | ); |
e8be2606 | 622 | resolved.def = InstanceDef::ReifyShim(def, reason); |
c295e0f8 | 623 | } |
136023e0 XL |
624 | } |
625 | } | |
626 | InstanceDef::Virtual(def_id, _) => { | |
627 | debug!(" => vtable fn pointer created for virtual call"); | |
e8be2606 | 628 | resolved.def = InstanceDef::ReifyShim(def_id, reason) |
136023e0 XL |
629 | } |
630 | _ => {} | |
631 | } | |
632 | ||
633 | resolved | |
634 | }) | |
a1dfa0c6 XL |
635 | } |
636 | } | |
637 | ||
ff7c6d11 | 638 | pub fn resolve_closure( |
dc9dc135 | 639 | tcx: TyCtxt<'tcx>, |
0bf4aa26 | 640 | def_id: DefId, |
add651ee | 641 | args: ty::GenericArgsRef<'tcx>, |
dc9dc135 | 642 | requested_kind: ty::ClosureKind, |
c0240ec0 | 643 | ) -> Instance<'tcx> { |
add651ee | 644 | let actual_kind = args.as_closure().kind(); |
ea8adc8c | 645 | |
ff7c6d11 | 646 | match needs_fn_once_adapter_shim(actual_kind, requested_kind) { |
add651ee | 647 | Ok(true) => Instance::fn_once_adapter_instance(tcx, def_id, args), |
c0240ec0 | 648 | _ => Instance::new(def_id, args), |
ff7c6d11 | 649 | } |
ea8adc8c | 650 | } |
a1dfa0c6 | 651 | |
dc9dc135 | 652 | pub fn resolve_drop_in_place(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> ty::Instance<'tcx> { |
3dfed10e | 653 | let def_id = tcx.require_lang_item(LangItem::DropInPlace, None); |
add651ee FG |
654 | let args = tcx.mk_args(&[ty.into()]); |
655 | Instance::expect_resolve(tcx, ty::ParamEnv::reveal_all(), def_id, args) | |
dc9dc135 XL |
656 | } |
657 | ||
e8be2606 FG |
658 | pub fn resolve_async_drop_in_place(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> ty::Instance<'tcx> { |
659 | let def_id = tcx.require_lang_item(LangItem::AsyncDropInPlace, None); | |
660 | let args = tcx.mk_args(&[ty.into()]); | |
661 | Instance::expect_resolve(tcx, ty::ParamEnv::reveal_all(), def_id, args) | |
662 | } | |
663 | ||
487cf647 | 664 | #[instrument(level = "debug", skip(tcx), ret)] |
dc9dc135 XL |
665 | pub fn fn_once_adapter_instance( |
666 | tcx: TyCtxt<'tcx>, | |
667 | closure_did: DefId, | |
add651ee | 668 | args: ty::GenericArgsRef<'tcx>, |
c0240ec0 | 669 | ) -> Instance<'tcx> { |
3dfed10e | 670 | let fn_once = tcx.require_lang_item(LangItem::FnOnce, None); |
dfeec247 XL |
671 | let call_once = tcx |
672 | .associated_items(fn_once) | |
74b04a01 | 673 | .in_definition_order() |
ba9703b0 | 674 | .find(|it| it.kind == ty::AssocKind::Fn) |
dfeec247 XL |
675 | .unwrap() |
676 | .def_id; | |
c295e0f8 XL |
677 | let track_caller = |
678 | tcx.codegen_fn_attrs(closure_did).flags.contains(CodegenFnAttrFlags::TRACK_CALLER); | |
679 | let def = ty::InstanceDef::ClosureOnceShim { call_once, track_caller }; | |
dc9dc135 | 680 | |
add651ee | 681 | let self_ty = Ty::new_closure(tcx, closure_did, args); |
dc9dc135 | 682 | |
c0240ec0 FG |
683 | let tupled_inputs_ty = args.as_closure().sig().map_bound(|sig| sig.inputs()[0]); |
684 | let tupled_inputs_ty = tcx.instantiate_bound_regions_with_erased(tupled_inputs_ty); | |
685 | let args = tcx.mk_args_trait(self_ty, [tupled_inputs_ty.into()]); | |
686 | ||
687 | debug!(?self_ty, args=?tupled_inputs_ty.tuple_fields()); | |
688 | Instance { def, args } | |
689 | } | |
690 | ||
691 | pub fn try_resolve_item_for_coroutine( | |
692 | tcx: TyCtxt<'tcx>, | |
693 | trait_item_id: DefId, | |
694 | trait_id: DefId, | |
695 | rcvr_args: ty::GenericArgsRef<'tcx>, | |
696 | ) -> Option<Instance<'tcx>> { | |
697 | let ty::Coroutine(coroutine_def_id, args) = *rcvr_args.type_at(0).kind() else { | |
698 | return None; | |
699 | }; | |
700 | let coroutine_kind = tcx.coroutine_kind(coroutine_def_id).unwrap(); | |
701 | ||
702 | let lang_items = tcx.lang_items(); | |
703 | let coroutine_callable_item = if Some(trait_id) == lang_items.future_trait() { | |
704 | assert_matches!( | |
705 | coroutine_kind, | |
706 | hir::CoroutineKind::Desugared(hir::CoroutineDesugaring::Async, _) | |
707 | ); | |
708 | hir::LangItem::FuturePoll | |
709 | } else if Some(trait_id) == lang_items.iterator_trait() { | |
710 | assert_matches!( | |
711 | coroutine_kind, | |
712 | hir::CoroutineKind::Desugared(hir::CoroutineDesugaring::Gen, _) | |
713 | ); | |
714 | hir::LangItem::IteratorNext | |
715 | } else if Some(trait_id) == lang_items.async_iterator_trait() { | |
716 | assert_matches!( | |
717 | coroutine_kind, | |
718 | hir::CoroutineKind::Desugared(hir::CoroutineDesugaring::AsyncGen, _) | |
719 | ); | |
720 | hir::LangItem::AsyncIteratorPollNext | |
721 | } else if Some(trait_id) == lang_items.coroutine_trait() { | |
722 | assert_matches!(coroutine_kind, hir::CoroutineKind::Coroutine(_)); | |
723 | hir::LangItem::CoroutineResume | |
724 | } else { | |
725 | return None; | |
726 | }; | |
dc9dc135 | 727 | |
c0240ec0 | 728 | if tcx.lang_items().get(coroutine_callable_item) == Some(trait_item_id) { |
c620b35d FG |
729 | let ty::Coroutine(_, id_args) = *tcx.type_of(coroutine_def_id).skip_binder().kind() |
730 | else { | |
731 | bug!() | |
732 | }; | |
733 | ||
734 | // If the closure's kind ty disagrees with the identity closure's kind ty, | |
735 | // then this must be a coroutine generated by one of the `ConstructCoroutineInClosureShim`s. | |
736 | if args.as_coroutine().kind_ty() == id_args.as_coroutine().kind_ty() { | |
737 | Some(Instance { def: ty::InstanceDef::Item(coroutine_def_id), args }) | |
738 | } else { | |
739 | Some(Instance { | |
e8be2606 | 740 | def: ty::InstanceDef::CoroutineKindShim { coroutine_def_id }, |
c620b35d FG |
741 | args, |
742 | }) | |
743 | } | |
c0240ec0 FG |
744 | } else { |
745 | // All other methods should be defaulted methods of the built-in trait. | |
746 | // This is important for `Iterator`'s combinators, but also useful for | |
747 | // adding future default methods to `Future`, for instance. | |
748 | debug_assert!(tcx.defaultness(trait_item_id).has_value()); | |
749 | Some(Instance::new(trait_item_id, rcvr_args)) | |
750 | } | |
dc9dc135 XL |
751 | } |
752 | ||
1b1a35ee | 753 | /// Depending on the kind of `InstanceDef`, the MIR body associated with an |
ba9703b0 | 754 | /// instance is expressed in terms of the generic parameters of `self.def_id()`, and in other |
c620b35d FG |
755 | /// cases the MIR body is expressed in terms of the types found in the generic parameter array. |
756 | /// In the former case, we want to instantiate those generic types and replace them with the | |
add651ee | 757 | /// values from the args when monomorphizing the function body. But in the latter case, we |
c620b35d | 758 | /// don't want to do that instantiation, since it has already been done effectively. |
ba9703b0 | 759 | /// |
add651ee | 760 | /// This function returns `Some(args)` in the former case and `None` otherwise -- i.e., if |
c620b35d | 761 | /// this function returns `None`, then the MIR body does not require instantiation during |
1b1a35ee | 762 | /// codegen. |
add651ee FG |
763 | fn args_for_mir_body(&self) -> Option<GenericArgsRef<'tcx>> { |
764 | self.def.has_polymorphic_mir_body().then_some(self.args) | |
a1dfa0c6 | 765 | } |
3dfed10e | 766 | |
31ef2f64 | 767 | pub fn instantiate_mir<T>(&self, tcx: TyCtxt<'tcx>, v: EarlyBinder<'tcx, &T>) -> T |
29967ef6 | 768 | where |
9ffffee4 | 769 | T: TypeFoldable<TyCtxt<'tcx>> + Copy, |
29967ef6 | 770 | { |
49aad941 | 771 | let v = v.map_bound(|v| *v); |
add651ee FG |
772 | if let Some(args) = self.args_for_mir_body() { |
773 | v.instantiate(tcx, args) | |
04454e1e | 774 | } else { |
add651ee | 775 | v.instantiate_identity() |
04454e1e | 776 | } |
29967ef6 XL |
777 | } |
778 | ||
cdc7bbd5 | 779 | #[inline(always)] |
c620b35d | 780 | // Keep me in sync with try_instantiate_mir_and_normalize_erasing_regions |
781aab86 | 781 | pub fn instantiate_mir_and_normalize_erasing_regions<T>( |
29967ef6 XL |
782 | &self, |
783 | tcx: TyCtxt<'tcx>, | |
784 | param_env: ty::ParamEnv<'tcx>, | |
31ef2f64 | 785 | v: EarlyBinder<'tcx, T>, |
29967ef6 XL |
786 | ) -> T |
787 | where | |
c620b35d | 788 | T: TypeFoldable<TyCtxt<'tcx>>, |
29967ef6 | 789 | { |
add651ee | 790 | if let Some(args) = self.args_for_mir_body() { |
781aab86 | 791 | tcx.instantiate_and_normalize_erasing_regions(args, param_env, v) |
29967ef6 | 792 | } else { |
c620b35d | 793 | tcx.normalize_erasing_regions(param_env, v.instantiate_identity()) |
29967ef6 XL |
794 | } |
795 | } | |
796 | ||
a2a8927a | 797 | #[inline(always)] |
c620b35d | 798 | // Keep me in sync with instantiate_mir_and_normalize_erasing_regions |
781aab86 | 799 | pub fn try_instantiate_mir_and_normalize_erasing_regions<T>( |
a2a8927a XL |
800 | &self, |
801 | tcx: TyCtxt<'tcx>, | |
802 | param_env: ty::ParamEnv<'tcx>, | |
31ef2f64 | 803 | v: EarlyBinder<'tcx, T>, |
a2a8927a XL |
804 | ) -> Result<T, NormalizationError<'tcx>> |
805 | where | |
ed00b5ec | 806 | T: TypeFoldable<TyCtxt<'tcx>>, |
a2a8927a | 807 | { |
add651ee | 808 | if let Some(args) = self.args_for_mir_body() { |
781aab86 | 809 | tcx.try_instantiate_and_normalize_erasing_regions(args, param_env, v) |
a2a8927a | 810 | } else { |
ed00b5ec FG |
811 | // We're using `instantiate_identity` as e.g. |
812 | // `FnPtrShim` is separately generated for every | |
813 | // instantiation of the `FnDef`, so the MIR body | |
814 | // is already instantiated. Any generic parameters it | |
815 | // contains are generic parameters from the caller. | |
816 | tcx.try_normalize_erasing_regions(param_env, v.instantiate_identity()) | |
a2a8927a XL |
817 | } |
818 | } | |
819 | ||
add651ee | 820 | /// Returns a new `Instance` where generic parameters in `instance.args` are replaced by |
6a06907d | 821 | /// identity parameters if they are determined to be unused in `instance.def`. |
3dfed10e XL |
822 | pub fn polymorphize(self, tcx: TyCtxt<'tcx>) -> Self { |
823 | debug!("polymorphize: running polymorphization analysis"); | |
064997fb | 824 | if !tcx.sess.opts.unstable_opts.polymorphize { |
3dfed10e XL |
825 | return self; |
826 | } | |
827 | ||
add651ee FG |
828 | let polymorphized_args = polymorphize(tcx, self.def, self.args); |
829 | debug!("polymorphize: self={:?} polymorphized_args={:?}", self, polymorphized_args); | |
830 | Self { def: self.def, args: polymorphized_args } | |
3dfed10e XL |
831 | } |
832 | } | |
833 | ||
834 | fn polymorphize<'tcx>( | |
835 | tcx: TyCtxt<'tcx>, | |
c295e0f8 | 836 | instance: ty::InstanceDef<'tcx>, |
add651ee FG |
837 | args: GenericArgsRef<'tcx>, |
838 | ) -> GenericArgsRef<'tcx> { | |
839 | debug!("polymorphize({:?}, {:?})", instance, args); | |
c295e0f8 | 840 | let unused = tcx.unused_generic_params(instance); |
3dfed10e XL |
841 | debug!("polymorphize: unused={:?}", unused); |
842 | ||
ed00b5ec | 843 | // If this is a closure or coroutine then we need to handle the case where another closure |
3dfed10e XL |
844 | // from the function is captured as an upvar and hasn't been polymorphized. In this case, |
845 | // the unpolymorphized upvar closure would result in a polymorphized closure producing | |
846 | // multiple mono items (and eventually symbol clashes). | |
c295e0f8 | 847 | let def_id = instance.def_id(); |
c0240ec0 FG |
848 | let upvars_ty = match tcx.type_of(def_id).skip_binder().kind() { |
849 | ty::Closure(..) => Some(args.as_closure().tupled_upvars_ty()), | |
c620b35d FG |
850 | ty::Coroutine(..) => { |
851 | assert_eq!( | |
852 | args.as_coroutine().kind_ty(), | |
853 | tcx.types.unit, | |
854 | "polymorphization does not support coroutines from async closures" | |
855 | ); | |
856 | Some(args.as_coroutine().tupled_upvars_ty()) | |
857 | } | |
c0240ec0 | 858 | _ => None, |
3dfed10e | 859 | }; |
49aad941 | 860 | let has_upvars = upvars_ty.is_some_and(|ty| !ty.tuple_fields().is_empty()); |
3dfed10e XL |
861 | debug!("polymorphize: upvars_ty={:?} has_upvars={:?}", upvars_ty, has_upvars); |
862 | ||
863 | struct PolymorphizationFolder<'tcx> { | |
864 | tcx: TyCtxt<'tcx>, | |
fc512014 | 865 | } |
3dfed10e | 866 | |
9ffffee4 FG |
867 | impl<'tcx> ty::TypeFolder<TyCtxt<'tcx>> for PolymorphizationFolder<'tcx> { |
868 | fn interner(&self) -> TyCtxt<'tcx> { | |
3dfed10e XL |
869 | self.tcx |
870 | } | |
871 | ||
872 | fn fold_ty(&mut self, ty: Ty<'tcx>) -> Ty<'tcx> { | |
873 | debug!("fold_ty: ty={:?}", ty); | |
5099ac24 | 874 | match *ty.kind() { |
add651ee FG |
875 | ty::Closure(def_id, args) => { |
876 | let polymorphized_args = | |
877 | polymorphize(self.tcx, ty::InstanceDef::Item(def_id), args); | |
878 | if args == polymorphized_args { | |
3dfed10e XL |
879 | ty |
880 | } else { | |
add651ee | 881 | Ty::new_closure(self.tcx, def_id, polymorphized_args) |
3dfed10e XL |
882 | } |
883 | } | |
c0240ec0 | 884 | ty::Coroutine(def_id, args) => { |
add651ee FG |
885 | let polymorphized_args = |
886 | polymorphize(self.tcx, ty::InstanceDef::Item(def_id), args); | |
887 | if args == polymorphized_args { | |
3dfed10e XL |
888 | ty |
889 | } else { | |
c0240ec0 | 890 | Ty::new_coroutine(self.tcx, def_id, polymorphized_args) |
3dfed10e XL |
891 | } |
892 | } | |
893 | _ => ty.super_fold_with(self), | |
894 | } | |
895 | } | |
896 | } | |
897 | ||
add651ee | 898 | GenericArgs::for_item(tcx, def_id, |param, _| { |
9c376795 | 899 | let is_unused = unused.is_unused(param.index); |
3dfed10e XL |
900 | debug!("polymorphize: param={:?} is_unused={:?}", param, is_unused); |
901 | match param.kind { | |
902 | // Upvar case: If parameter is a type parameter.. | |
903 | ty::GenericParamDefKind::Type { .. } if | |
904 | // ..and has upvars.. | |
905 | has_upvars && | |
906 | // ..and this param has the same type as the tupled upvars.. | |
add651ee | 907 | upvars_ty == Some(args[param.index as usize].expect_ty()) => { |
3dfed10e XL |
908 | // ..then double-check that polymorphization marked it used.. |
909 | debug_assert!(!is_unused); | |
ed00b5ec | 910 | // ..and polymorphize any closures/coroutines captured as upvars. |
3dfed10e XL |
911 | let upvars_ty = upvars_ty.unwrap(); |
912 | let polymorphized_upvars_ty = upvars_ty.fold_with( | |
913 | &mut PolymorphizationFolder { tcx }); | |
914 | debug!("polymorphize: polymorphized_upvars_ty={:?}", polymorphized_upvars_ty); | |
915 | ty::GenericArg::from(polymorphized_upvars_ty) | |
916 | }, | |
917 | ||
918 | // Simple case: If parameter is a const or type parameter.. | |
cdc7bbd5 | 919 | ty::GenericParamDefKind::Const { .. } | ty::GenericParamDefKind::Type { .. } if |
3dfed10e | 920 | // ..and is within range and unused.. |
9c376795 | 921 | unused.is_unused(param.index) => |
3dfed10e XL |
922 | // ..then use the identity for this parameter. |
923 | tcx.mk_param_from_def(param), | |
924 | ||
925 | // Otherwise, use the parameter as before. | |
add651ee | 926 | _ => args[param.index as usize], |
3dfed10e XL |
927 | } |
928 | }) | |
ea8adc8c XL |
929 | } |
930 | ||
dc9dc135 XL |
931 | fn needs_fn_once_adapter_shim( |
932 | actual_closure_kind: ty::ClosureKind, | |
933 | trait_closure_kind: ty::ClosureKind, | |
934 | ) -> Result<bool, ()> { | |
ea8adc8c | 935 | match (actual_closure_kind, trait_closure_kind) { |
dfeec247 XL |
936 | (ty::ClosureKind::Fn, ty::ClosureKind::Fn) |
937 | | (ty::ClosureKind::FnMut, ty::ClosureKind::FnMut) | |
938 | | (ty::ClosureKind::FnOnce, ty::ClosureKind::FnOnce) => { | |
939 | // No adapter needed. | |
940 | Ok(false) | |
941 | } | |
ea8adc8c | 942 | (ty::ClosureKind::Fn, ty::ClosureKind::FnMut) => { |
9c376795 | 943 | // The closure fn `llfn` is a `fn(&self, ...)`. We want a |
94b46f34 | 944 | // `fn(&mut self, ...)`. In fact, at codegen time, these are |
ea8adc8c XL |
945 | // basically the same thing, so we can just return llfn. |
946 | Ok(false) | |
947 | } | |
ba9703b0 | 948 | (ty::ClosureKind::Fn | ty::ClosureKind::FnMut, ty::ClosureKind::FnOnce) => { |
dfeec247 | 949 | // The closure fn `llfn` is a `fn(&self, ...)` or `fn(&mut |
9c376795 | 950 | // self, ...)`. We want a `fn(self, ...)`. We can produce |
dfeec247 XL |
951 | // this by doing something like: |
952 | // | |
953 | // fn call_once(self, ...) { call_mut(&self, ...) } | |
954 | // fn call_once(mut self, ...) { call_mut(&mut self, ...) } | |
955 | // | |
956 | // These are both the same at codegen time. | |
957 | Ok(true) | |
ea8adc8c | 958 | } |
ba9703b0 | 959 | (ty::ClosureKind::FnMut | ty::ClosureKind::FnOnce, _) => Err(()), |
ea8adc8c XL |
960 | } |
961 | } | |
9c376795 FG |
962 | |
963 | // Set bits represent unused generic parameters. | |
964 | // An empty set indicates that all parameters are used. | |
965 | #[derive(Debug, Copy, Clone, Eq, PartialEq, Decodable, Encodable, HashStable)] | |
966 | pub struct UnusedGenericParams(FiniteBitSet<u32>); | |
967 | ||
353b0b11 FG |
968 | impl Default for UnusedGenericParams { |
969 | fn default() -> Self { | |
970 | UnusedGenericParams::new_all_used() | |
971 | } | |
972 | } | |
973 | ||
9c376795 FG |
974 | impl UnusedGenericParams { |
975 | pub fn new_all_unused(amount: u32) -> Self { | |
976 | let mut bitset = FiniteBitSet::new_empty(); | |
977 | bitset.set_range(0..amount); | |
978 | Self(bitset) | |
979 | } | |
980 | ||
981 | pub fn new_all_used() -> Self { | |
982 | Self(FiniteBitSet::new_empty()) | |
983 | } | |
984 | ||
985 | pub fn mark_used(&mut self, idx: u32) { | |
986 | self.0.clear(idx); | |
987 | } | |
988 | ||
989 | pub fn is_unused(&self, idx: u32) -> bool { | |
990 | self.0.contains(idx).unwrap_or(false) | |
991 | } | |
992 | ||
993 | pub fn is_used(&self, idx: u32) -> bool { | |
994 | !self.is_unused(idx) | |
995 | } | |
996 | ||
997 | pub fn all_used(&self) -> bool { | |
998 | self.0.is_empty() | |
999 | } | |
353b0b11 FG |
1000 | |
1001 | pub fn bits(&self) -> u32 { | |
1002 | self.0.0 | |
1003 | } | |
1004 | ||
1005 | pub fn from_bits(bits: u32) -> UnusedGenericParams { | |
1006 | UnusedGenericParams(FiniteBitSet(bits)) | |
1007 | } | |
9c376795 | 1008 | } |