]>
Commit | Line | Data |
---|---|---|
2b03887a FG |
1 | use rustc_hir as hir; |
2 | use rustc_hir::lang_items::LangItem; | |
3 | use rustc_middle::ty::layout::{ | |
4 | fn_can_unwind, FnAbiError, HasParamEnv, HasTyCtxt, LayoutCx, LayoutOf, TyAndLayout, | |
5 | }; | |
353b0b11 | 6 | use rustc_middle::ty::{self, InstanceDef, Ty, TyCtxt}; |
2b03887a FG |
7 | use rustc_session::config::OptLevel; |
8 | use rustc_span::def_id::DefId; | |
9 | use rustc_target::abi::call::{ | |
10 | ArgAbi, ArgAttribute, ArgAttributes, ArgExtension, Conv, FnAbi, PassMode, Reg, RegKind, | |
11 | }; | |
12 | use rustc_target::abi::*; | |
13 | use rustc_target::spec::abi::Abi as SpecAbi; | |
14 | ||
15 | use std::iter; | |
16 | ||
17 | pub fn provide(providers: &mut ty::query::Providers) { | |
18 | *providers = ty::query::Providers { fn_abi_of_fn_ptr, fn_abi_of_instance, ..*providers }; | |
19 | } | |
20 | ||
21 | // NOTE(eddyb) this is private to avoid using it from outside of | |
22 | // `fn_abi_of_instance` - any other uses are either too high-level | |
23 | // for `Instance` (e.g. typeck would use `Ty::fn_sig` instead), | |
24 | // or should go through `FnAbi` instead, to avoid losing any | |
25 | // adjustments `fn_abi_of_instance` might be performing. | |
26 | #[tracing::instrument(level = "debug", skip(tcx, param_env))] | |
27 | fn fn_sig_for_fn_abi<'tcx>( | |
28 | tcx: TyCtxt<'tcx>, | |
29 | instance: ty::Instance<'tcx>, | |
30 | param_env: ty::ParamEnv<'tcx>, | |
31 | ) -> ty::PolyFnSig<'tcx> { | |
353b0b11 FG |
32 | if let InstanceDef::ThreadLocalShim(..) = instance.def { |
33 | return ty::Binder::dummy(tcx.mk_fn_sig( | |
34 | [], | |
35 | tcx.thread_local_ptr_ty(instance.def_id()), | |
36 | false, | |
37 | hir::Unsafety::Normal, | |
38 | rustc_target::spec::abi::Abi::Unadjusted, | |
39 | )); | |
40 | } | |
41 | ||
2b03887a FG |
42 | let ty = instance.ty(tcx, param_env); |
43 | match *ty.kind() { | |
44 | ty::FnDef(..) => { | |
45 | // HACK(davidtwco,eddyb): This is a workaround for polymorphization considering | |
46 | // parameters unused if they show up in the signature, but not in the `mir::Body` | |
47 | // (i.e. due to being inside a projection that got normalized, see | |
9c376795 | 48 | // `tests/ui/polymorphization/normalized_sig_types.rs`), and codegen not keeping |
2b03887a FG |
49 | // track of a polymorphization `ParamEnv` to allow normalizing later. |
50 | // | |
51 | // We normalize the `fn_sig` again after substituting at a later point. | |
52 | let mut sig = match *ty.kind() { | |
53 | ty::FnDef(def_id, substs) => tcx | |
9ffffee4 | 54 | .fn_sig(def_id) |
2b03887a FG |
55 | .map_bound(|fn_sig| { |
56 | tcx.normalize_erasing_regions(tcx.param_env(def_id), fn_sig) | |
57 | }) | |
58 | .subst(tcx, substs), | |
59 | _ => unreachable!(), | |
60 | }; | |
61 | ||
62 | if let ty::InstanceDef::VTableShim(..) = instance.def { | |
63 | // Modify `fn(self, ...)` to `fn(self: *mut Self, ...)`. | |
64 | sig = sig.map_bound(|mut sig| { | |
65 | let mut inputs_and_output = sig.inputs_and_output.to_vec(); | |
66 | inputs_and_output[0] = tcx.mk_mut_ptr(inputs_and_output[0]); | |
9ffffee4 | 67 | sig.inputs_and_output = tcx.mk_type_list(&inputs_and_output); |
2b03887a FG |
68 | sig |
69 | }); | |
70 | } | |
71 | sig | |
72 | } | |
73 | ty::Closure(def_id, substs) => { | |
74 | let sig = substs.as_closure().sig(); | |
75 | ||
9ffffee4 | 76 | let bound_vars = tcx.mk_bound_variable_kinds_from_iter( |
2b03887a FG |
77 | sig.bound_vars().iter().chain(iter::once(ty::BoundVariableKind::Region(ty::BrEnv))), |
78 | ); | |
79 | let br = ty::BoundRegion { | |
80 | var: ty::BoundVar::from_usize(bound_vars.len() - 1), | |
81 | kind: ty::BoundRegionKind::BrEnv, | |
82 | }; | |
9ffffee4 | 83 | let env_region = tcx.mk_re_late_bound(ty::INNERMOST, br); |
2b03887a FG |
84 | let env_ty = tcx.closure_env_ty(def_id, substs, env_region).unwrap(); |
85 | ||
86 | let sig = sig.skip_binder(); | |
87 | ty::Binder::bind_with_vars( | |
88 | tcx.mk_fn_sig( | |
89 | iter::once(env_ty).chain(sig.inputs().iter().cloned()), | |
90 | sig.output(), | |
91 | sig.c_variadic, | |
92 | sig.unsafety, | |
93 | sig.abi, | |
94 | ), | |
95 | bound_vars, | |
96 | ) | |
97 | } | |
9c376795 | 98 | ty::Generator(did, substs, _) => { |
2b03887a FG |
99 | let sig = substs.as_generator().poly_sig(); |
100 | ||
9ffffee4 | 101 | let bound_vars = tcx.mk_bound_variable_kinds_from_iter( |
2b03887a FG |
102 | sig.bound_vars().iter().chain(iter::once(ty::BoundVariableKind::Region(ty::BrEnv))), |
103 | ); | |
104 | let br = ty::BoundRegion { | |
105 | var: ty::BoundVar::from_usize(bound_vars.len() - 1), | |
106 | kind: ty::BoundRegionKind::BrEnv, | |
107 | }; | |
9ffffee4 | 108 | let env_ty = tcx.mk_mut_ref(tcx.mk_re_late_bound(ty::INNERMOST, br), ty); |
2b03887a FG |
109 | |
110 | let pin_did = tcx.require_lang_item(LangItem::Pin, None); | |
111 | let pin_adt_ref = tcx.adt_def(pin_did); | |
9ffffee4 | 112 | let pin_substs = tcx.mk_substs(&[env_ty.into()]); |
2b03887a FG |
113 | let env_ty = tcx.mk_adt(pin_adt_ref, pin_substs); |
114 | ||
115 | let sig = sig.skip_binder(); | |
9c376795 FG |
116 | // The `FnSig` and the `ret_ty` here is for a generators main |
117 | // `Generator::resume(...) -> GeneratorState` function in case we | |
118 | // have an ordinary generator, or the `Future::poll(...) -> Poll` | |
119 | // function in case this is a special generator backing an async construct. | |
120 | let (resume_ty, ret_ty) = if tcx.generator_is_async(did) { | |
121 | // The signature should be `Future::poll(_, &mut Context<'_>) -> Poll<Output>` | |
122 | let poll_did = tcx.require_lang_item(LangItem::Poll, None); | |
123 | let poll_adt_ref = tcx.adt_def(poll_did); | |
9ffffee4 | 124 | let poll_substs = tcx.mk_substs(&[sig.return_ty.into()]); |
9c376795 FG |
125 | let ret_ty = tcx.mk_adt(poll_adt_ref, poll_substs); |
126 | ||
127 | // We have to replace the `ResumeTy` that is used for type and borrow checking | |
128 | // with `&mut Context<'_>` which is used in codegen. | |
129 | #[cfg(debug_assertions)] | |
130 | { | |
131 | if let ty::Adt(resume_ty_adt, _) = sig.resume_ty.kind() { | |
132 | let expected_adt = | |
133 | tcx.adt_def(tcx.require_lang_item(LangItem::ResumeTy, None)); | |
134 | assert_eq!(*resume_ty_adt, expected_adt); | |
135 | } else { | |
136 | panic!("expected `ResumeTy`, found `{:?}`", sig.resume_ty); | |
137 | }; | |
138 | } | |
139 | let context_mut_ref = tcx.mk_task_context(); | |
140 | ||
141 | (context_mut_ref, ret_ty) | |
142 | } else { | |
143 | // The signature should be `Generator::resume(_, Resume) -> GeneratorState<Yield, Return>` | |
144 | let state_did = tcx.require_lang_item(LangItem::GeneratorState, None); | |
145 | let state_adt_ref = tcx.adt_def(state_did); | |
9ffffee4 | 146 | let state_substs = tcx.mk_substs(&[sig.yield_ty.into(), sig.return_ty.into()]); |
9c376795 FG |
147 | let ret_ty = tcx.mk_adt(state_adt_ref, state_substs); |
148 | ||
149 | (sig.resume_ty, ret_ty) | |
150 | }; | |
151 | ||
2b03887a FG |
152 | ty::Binder::bind_with_vars( |
153 | tcx.mk_fn_sig( | |
9ffffee4 FG |
154 | [env_ty, resume_ty], |
155 | ret_ty, | |
2b03887a FG |
156 | false, |
157 | hir::Unsafety::Normal, | |
158 | rustc_target::spec::abi::Abi::Rust, | |
159 | ), | |
160 | bound_vars, | |
161 | ) | |
162 | } | |
163 | _ => bug!("unexpected type {:?} in Instance::fn_sig", ty), | |
164 | } | |
165 | } | |
166 | ||
167 | #[inline] | |
168 | fn conv_from_spec_abi(tcx: TyCtxt<'_>, abi: SpecAbi) -> Conv { | |
169 | use rustc_target::spec::abi::Abi::*; | |
170 | match tcx.sess.target.adjust_abi(abi) { | |
171 | RustIntrinsic | PlatformIntrinsic | Rust | RustCall => Conv::Rust, | |
172 | RustCold => Conv::RustCold, | |
173 | ||
174 | // It's the ABI's job to select this, not ours. | |
175 | System { .. } => bug!("system abi should be selected elsewhere"), | |
176 | EfiApi => bug!("eficall abi should be selected elsewhere"), | |
177 | ||
178 | Stdcall { .. } => Conv::X86Stdcall, | |
179 | Fastcall { .. } => Conv::X86Fastcall, | |
180 | Vectorcall { .. } => Conv::X86VectorCall, | |
181 | Thiscall { .. } => Conv::X86ThisCall, | |
182 | C { .. } => Conv::C, | |
183 | Unadjusted => Conv::C, | |
184 | Win64 { .. } => Conv::X86_64Win64, | |
185 | SysV64 { .. } => Conv::X86_64SysV, | |
186 | Aapcs { .. } => Conv::ArmAapcs, | |
187 | CCmseNonSecureCall => Conv::CCmseNonSecureCall, | |
188 | PtxKernel => Conv::PtxKernel, | |
189 | Msp430Interrupt => Conv::Msp430Intr, | |
190 | X86Interrupt => Conv::X86Intr, | |
191 | AmdGpuKernel => Conv::AmdGpuKernel, | |
192 | AvrInterrupt => Conv::AvrInterrupt, | |
193 | AvrNonBlockingInterrupt => Conv::AvrNonBlockingInterrupt, | |
194 | Wasm => Conv::C, | |
195 | ||
196 | // These API constants ought to be more specific... | |
197 | Cdecl { .. } => Conv::C, | |
198 | } | |
199 | } | |
200 | ||
201 | fn fn_abi_of_fn_ptr<'tcx>( | |
202 | tcx: TyCtxt<'tcx>, | |
203 | query: ty::ParamEnvAnd<'tcx, (ty::PolyFnSig<'tcx>, &'tcx ty::List<Ty<'tcx>>)>, | |
204 | ) -> Result<&'tcx FnAbi<'tcx, Ty<'tcx>>, FnAbiError<'tcx>> { | |
205 | let (param_env, (sig, extra_args)) = query.into_parts(); | |
206 | ||
207 | let cx = LayoutCx { tcx, param_env }; | |
208 | fn_abi_new_uncached(&cx, sig, extra_args, None, None, false) | |
209 | } | |
210 | ||
211 | fn fn_abi_of_instance<'tcx>( | |
212 | tcx: TyCtxt<'tcx>, | |
213 | query: ty::ParamEnvAnd<'tcx, (ty::Instance<'tcx>, &'tcx ty::List<Ty<'tcx>>)>, | |
214 | ) -> Result<&'tcx FnAbi<'tcx, Ty<'tcx>>, FnAbiError<'tcx>> { | |
215 | let (param_env, (instance, extra_args)) = query.into_parts(); | |
216 | ||
217 | let sig = fn_sig_for_fn_abi(tcx, instance, param_env); | |
218 | ||
9ffffee4 FG |
219 | let caller_location = |
220 | instance.def.requires_caller_location(tcx).then(|| tcx.caller_location_ty()); | |
2b03887a FG |
221 | |
222 | fn_abi_new_uncached( | |
223 | &LayoutCx { tcx, param_env }, | |
224 | sig, | |
225 | extra_args, | |
226 | caller_location, | |
227 | Some(instance.def_id()), | |
228 | matches!(instance.def, ty::InstanceDef::Virtual(..)), | |
229 | ) | |
230 | } | |
231 | ||
232 | // Handle safe Rust thin and fat pointers. | |
233 | fn adjust_for_rust_scalar<'tcx>( | |
234 | cx: LayoutCx<'tcx, TyCtxt<'tcx>>, | |
235 | attrs: &mut ArgAttributes, | |
236 | scalar: Scalar, | |
237 | layout: TyAndLayout<'tcx>, | |
238 | offset: Size, | |
239 | is_return: bool, | |
240 | ) { | |
241 | // Booleans are always a noundef i1 that needs to be zero-extended. | |
242 | if scalar.is_bool() { | |
243 | attrs.ext(ArgExtension::Zext); | |
244 | attrs.set(ArgAttribute::NoUndef); | |
245 | return; | |
246 | } | |
247 | ||
9c376795 | 248 | if !scalar.is_uninit_valid() { |
2b03887a FG |
249 | attrs.set(ArgAttribute::NoUndef); |
250 | } | |
251 | ||
252 | // Only pointer types handled below. | |
9ffffee4 | 253 | let Scalar::Initialized { value: Pointer(_), valid_range} = scalar else { return }; |
2b03887a FG |
254 | |
255 | if !valid_range.contains(0) { | |
256 | attrs.set(ArgAttribute::NonNull); | |
257 | } | |
258 | ||
259 | if let Some(pointee) = layout.pointee_info_at(&cx, offset) { | |
260 | if let Some(kind) = pointee.safe { | |
261 | attrs.pointee_align = Some(pointee.align); | |
262 | ||
9ffffee4 FG |
263 | // `Box` are not necessarily dereferenceable for the entire duration of the function as |
264 | // they can be deallocated at any time. Same for non-frozen shared references (see | |
265 | // <https://github.com/rust-lang/rust/pull/98017>), and for mutable references to | |
266 | // potentially self-referential types (see | |
267 | // <https://github.com/rust-lang/unsafe-code-guidelines/issues/381>). If LLVM had a way | |
268 | // to say "dereferenceable on entry" we could use it here. | |
2b03887a | 269 | attrs.pointee_size = match kind { |
9ffffee4 FG |
270 | PointerKind::Box { .. } |
271 | | PointerKind::SharedRef { frozen: false } | |
272 | | PointerKind::MutableRef { unpin: false } => Size::ZERO, | |
273 | PointerKind::SharedRef { frozen: true } | |
274 | | PointerKind::MutableRef { unpin: true } => pointee.size, | |
2b03887a FG |
275 | }; |
276 | ||
2b03887a FG |
277 | // The aliasing rules for `Box<T>` are still not decided, but currently we emit |
278 | // `noalias` for it. This can be turned off using an unstable flag. | |
279 | // See https://github.com/rust-lang/unsafe-code-guidelines/issues/326 | |
9c376795 FG |
280 | let noalias_for_box = cx.tcx.sess.opts.unstable_opts.box_noalias; |
281 | ||
282 | // LLVM prior to version 12 had known miscompiles in the presence of noalias attributes | |
283 | // (see #54878), so it was conditionally disabled, but we don't support earlier | |
284 | // versions at all anymore. We still support turning it off using -Zmutable-noalias. | |
285 | let noalias_mut_ref = cx.tcx.sess.opts.unstable_opts.mutable_noalias; | |
2b03887a | 286 | |
9ffffee4 FG |
287 | // `&T` where `T` contains no `UnsafeCell<U>` is immutable, and can be marked as both |
288 | // `readonly` and `noalias`, as LLVM's definition of `noalias` is based solely on memory | |
289 | // dependencies rather than pointer equality. However this only applies to arguments, | |
290 | // not return values. | |
2b03887a | 291 | // |
9ffffee4 | 292 | // `&mut T` and `Box<T>` where `T: Unpin` are unique and hence `noalias`. |
2b03887a | 293 | let no_alias = match kind { |
9ffffee4 FG |
294 | PointerKind::SharedRef { frozen } => frozen, |
295 | PointerKind::MutableRef { unpin } => unpin && noalias_mut_ref, | |
296 | PointerKind::Box { unpin } => unpin && noalias_for_box, | |
2b03887a | 297 | }; |
9c376795 FG |
298 | // We can never add `noalias` in return position; that LLVM attribute has some very surprising semantics |
299 | // (see <https://github.com/rust-lang/unsafe-code-guidelines/issues/385#issuecomment-1368055745>). | |
300 | if no_alias && !is_return { | |
2b03887a FG |
301 | attrs.set(ArgAttribute::NoAlias); |
302 | } | |
303 | ||
9ffffee4 | 304 | if matches!(kind, PointerKind::SharedRef { frozen: true }) && !is_return { |
2b03887a FG |
305 | attrs.set(ArgAttribute::ReadOnly); |
306 | } | |
2b03887a FG |
307 | } |
308 | } | |
309 | } | |
310 | ||
311 | // FIXME(eddyb) perhaps group the signature/type-containing (or all of them?) | |
312 | // arguments of this method, into a separate `struct`. | |
313 | #[tracing::instrument(level = "debug", skip(cx, caller_location, fn_def_id, force_thin_self_ptr))] | |
314 | fn fn_abi_new_uncached<'tcx>( | |
315 | cx: &LayoutCx<'tcx, TyCtxt<'tcx>>, | |
316 | sig: ty::PolyFnSig<'tcx>, | |
317 | extra_args: &[Ty<'tcx>], | |
318 | caller_location: Option<Ty<'tcx>>, | |
319 | fn_def_id: Option<DefId>, | |
320 | // FIXME(eddyb) replace this with something typed, like an `enum`. | |
321 | force_thin_self_ptr: bool, | |
322 | ) -> Result<&'tcx FnAbi<'tcx, Ty<'tcx>>, FnAbiError<'tcx>> { | |
323 | let sig = cx.tcx.normalize_erasing_late_bound_regions(cx.param_env, sig); | |
324 | ||
325 | let conv = conv_from_spec_abi(cx.tcx(), sig.abi); | |
326 | ||
327 | let mut inputs = sig.inputs(); | |
328 | let extra_args = if sig.abi == RustCall { | |
329 | assert!(!sig.c_variadic && extra_args.is_empty()); | |
330 | ||
331 | if let Some(input) = sig.inputs().last() { | |
332 | if let ty::Tuple(tupled_arguments) = input.kind() { | |
333 | inputs = &sig.inputs()[0..sig.inputs().len() - 1]; | |
334 | tupled_arguments | |
335 | } else { | |
336 | bug!( | |
337 | "argument to function with \"rust-call\" ABI \ | |
338 | is not a tuple" | |
339 | ); | |
340 | } | |
341 | } else { | |
342 | bug!( | |
343 | "argument to function with \"rust-call\" ABI \ | |
344 | is not a tuple" | |
345 | ); | |
346 | } | |
347 | } else { | |
348 | assert!(sig.c_variadic || extra_args.is_empty()); | |
349 | extra_args | |
350 | }; | |
351 | ||
352 | let target = &cx.tcx.sess.target; | |
353 | let target_env_gnu_like = matches!(&target.env[..], "gnu" | "musl" | "uclibc"); | |
354 | let win_x64_gnu = target.os == "windows" && target.arch == "x86_64" && target.env == "gnu"; | |
355 | let linux_s390x_gnu_like = | |
356 | target.os == "linux" && target.arch == "s390x" && target_env_gnu_like; | |
357 | let linux_sparc64_gnu_like = | |
358 | target.os == "linux" && target.arch == "sparc64" && target_env_gnu_like; | |
359 | let linux_powerpc_gnu_like = | |
360 | target.os == "linux" && target.arch == "powerpc" && target_env_gnu_like; | |
361 | use SpecAbi::*; | |
362 | let rust_abi = matches!(sig.abi, RustIntrinsic | PlatformIntrinsic | Rust | RustCall); | |
363 | ||
364 | let arg_of = |ty: Ty<'tcx>, arg_idx: Option<usize>| -> Result<_, FnAbiError<'tcx>> { | |
365 | let span = tracing::debug_span!("arg_of"); | |
366 | let _entered = span.enter(); | |
367 | let is_return = arg_idx.is_none(); | |
368 | ||
369 | let layout = cx.layout_of(ty)?; | |
370 | let layout = if force_thin_self_ptr && arg_idx == Some(0) { | |
371 | // Don't pass the vtable, it's not an argument of the virtual fn. | |
372 | // Instead, pass just the data pointer, but give it the type `*const/mut dyn Trait` | |
373 | // or `&/&mut dyn Trait` because this is special-cased elsewhere in codegen | |
374 | make_thin_self_ptr(cx, layout) | |
375 | } else { | |
376 | layout | |
377 | }; | |
378 | ||
379 | let mut arg = ArgAbi::new(cx, layout, |layout, scalar, offset| { | |
380 | let mut attrs = ArgAttributes::new(); | |
381 | adjust_for_rust_scalar(*cx, &mut attrs, scalar, *layout, offset, is_return); | |
382 | attrs | |
383 | }); | |
384 | ||
385 | if arg.layout.is_zst() { | |
386 | // For some forsaken reason, x86_64-pc-windows-gnu | |
387 | // doesn't ignore zero-sized struct arguments. | |
388 | // The same is true for {s390x,sparc64,powerpc}-unknown-linux-{gnu,musl,uclibc}. | |
389 | if is_return | |
390 | || rust_abi | |
391 | || (!win_x64_gnu | |
392 | && !linux_s390x_gnu_like | |
393 | && !linux_sparc64_gnu_like | |
394 | && !linux_powerpc_gnu_like) | |
395 | { | |
396 | arg.mode = PassMode::Ignore; | |
397 | } | |
398 | } | |
399 | ||
400 | Ok(arg) | |
401 | }; | |
402 | ||
403 | let mut fn_abi = FnAbi { | |
404 | ret: arg_of(sig.output(), None)?, | |
405 | args: inputs | |
406 | .iter() | |
407 | .copied() | |
408 | .chain(extra_args.iter().copied()) | |
409 | .chain(caller_location) | |
410 | .enumerate() | |
411 | .map(|(i, ty)| arg_of(ty, Some(i))) | |
412 | .collect::<Result<_, _>>()?, | |
413 | c_variadic: sig.c_variadic, | |
414 | fixed_count: inputs.len() as u32, | |
415 | conv, | |
416 | can_unwind: fn_can_unwind(cx.tcx(), fn_def_id, sig.abi), | |
417 | }; | |
418 | fn_abi_adjust_for_abi(cx, &mut fn_abi, sig.abi, fn_def_id)?; | |
419 | debug!("fn_abi_new_uncached = {:?}", fn_abi); | |
420 | Ok(cx.tcx.arena.alloc(fn_abi)) | |
421 | } | |
422 | ||
423 | #[tracing::instrument(level = "trace", skip(cx))] | |
424 | fn fn_abi_adjust_for_abi<'tcx>( | |
425 | cx: &LayoutCx<'tcx, TyCtxt<'tcx>>, | |
426 | fn_abi: &mut FnAbi<'tcx, Ty<'tcx>>, | |
427 | abi: SpecAbi, | |
428 | fn_def_id: Option<DefId>, | |
429 | ) -> Result<(), FnAbiError<'tcx>> { | |
430 | if abi == SpecAbi::Unadjusted { | |
431 | return Ok(()); | |
432 | } | |
433 | ||
434 | if abi == SpecAbi::Rust | |
435 | || abi == SpecAbi::RustCall | |
436 | || abi == SpecAbi::RustIntrinsic | |
437 | || abi == SpecAbi::PlatformIntrinsic | |
438 | { | |
439 | // Look up the deduced parameter attributes for this function, if we have its def ID and | |
440 | // we're optimizing in non-incremental mode. We'll tag its parameters with those attributes | |
441 | // as appropriate. | |
442 | let deduced_param_attrs = if cx.tcx.sess.opts.optimize != OptLevel::No | |
443 | && cx.tcx.sess.opts.incremental.is_none() | |
444 | { | |
445 | fn_def_id.map(|fn_def_id| cx.tcx.deduced_param_attrs(fn_def_id)).unwrap_or_default() | |
446 | } else { | |
447 | &[] | |
448 | }; | |
449 | ||
450 | let fixup = |arg: &mut ArgAbi<'tcx, Ty<'tcx>>, arg_idx: Option<usize>| { | |
451 | if arg.is_ignore() { | |
452 | return; | |
453 | } | |
454 | ||
455 | match arg.layout.abi { | |
456 | Abi::Aggregate { .. } => {} | |
457 | ||
458 | // This is a fun case! The gist of what this is doing is | |
459 | // that we want callers and callees to always agree on the | |
460 | // ABI of how they pass SIMD arguments. If we were to *not* | |
461 | // make these arguments indirect then they'd be immediates | |
462 | // in LLVM, which means that they'd used whatever the | |
463 | // appropriate ABI is for the callee and the caller. That | |
464 | // means, for example, if the caller doesn't have AVX | |
465 | // enabled but the callee does, then passing an AVX argument | |
466 | // across this boundary would cause corrupt data to show up. | |
467 | // | |
468 | // This problem is fixed by unconditionally passing SIMD | |
469 | // arguments through memory between callers and callees | |
470 | // which should get them all to agree on ABI regardless of | |
471 | // target feature sets. Some more information about this | |
472 | // issue can be found in #44367. | |
473 | // | |
474 | // Note that the platform intrinsic ABI is exempt here as | |
475 | // that's how we connect up to LLVM and it's unstable | |
476 | // anyway, we control all calls to it in libstd. | |
477 | Abi::Vector { .. } | |
478 | if abi != SpecAbi::PlatformIntrinsic | |
479 | && cx.tcx.sess.target.simd_types_indirect => | |
480 | { | |
481 | arg.make_indirect(); | |
482 | return; | |
483 | } | |
484 | ||
485 | _ => return, | |
486 | } | |
487 | ||
488 | let size = arg.layout.size; | |
9ffffee4 | 489 | if arg.layout.is_unsized() || size > Pointer(AddressSpace::DATA).size(cx) { |
2b03887a FG |
490 | arg.make_indirect(); |
491 | } else { | |
492 | // We want to pass small aggregates as immediates, but using | |
493 | // a LLVM aggregate type for this leads to bad optimizations, | |
494 | // so we pick an appropriately sized integer type instead. | |
495 | arg.cast_to(Reg { kind: RegKind::Integer, size }); | |
496 | } | |
497 | ||
498 | // If we deduced that this parameter was read-only, add that to the attribute list now. | |
499 | // | |
500 | // The `readonly` parameter only applies to pointers, so we can only do this if the | |
501 | // argument was passed indirectly. (If the argument is passed directly, it's an SSA | |
502 | // value, so it's implicitly immutable.) | |
503 | if let (Some(arg_idx), &mut PassMode::Indirect { ref mut attrs, .. }) = | |
504 | (arg_idx, &mut arg.mode) | |
505 | { | |
506 | // The `deduced_param_attrs` list could be empty if this is a type of function | |
507 | // we can't deduce any parameters for, so make sure the argument index is in | |
508 | // bounds. | |
509 | if let Some(deduced_param_attrs) = deduced_param_attrs.get(arg_idx) { | |
510 | if deduced_param_attrs.read_only { | |
511 | attrs.regular.insert(ArgAttribute::ReadOnly); | |
512 | debug!("added deduced read-only attribute"); | |
513 | } | |
514 | } | |
515 | } | |
516 | }; | |
517 | ||
518 | fixup(&mut fn_abi.ret, None); | |
519 | for (arg_idx, arg) in fn_abi.args.iter_mut().enumerate() { | |
520 | fixup(arg, Some(arg_idx)); | |
521 | } | |
522 | } else { | |
523 | fn_abi.adjust_for_foreign_abi(cx, abi)?; | |
524 | } | |
525 | ||
526 | Ok(()) | |
527 | } | |
528 | ||
529 | #[tracing::instrument(level = "debug", skip(cx))] | |
530 | fn make_thin_self_ptr<'tcx>( | |
531 | cx: &(impl HasTyCtxt<'tcx> + HasParamEnv<'tcx>), | |
532 | layout: TyAndLayout<'tcx>, | |
533 | ) -> TyAndLayout<'tcx> { | |
534 | let tcx = cx.tcx(); | |
535 | let fat_pointer_ty = if layout.is_unsized() { | |
536 | // unsized `self` is passed as a pointer to `self` | |
537 | // FIXME (mikeyhew) change this to use &own if it is ever added to the language | |
538 | tcx.mk_mut_ptr(layout.ty) | |
539 | } else { | |
540 | match layout.abi { | |
541 | Abi::ScalarPair(..) | Abi::Scalar(..) => (), | |
542 | _ => bug!("receiver type has unsupported layout: {:?}", layout), | |
543 | } | |
544 | ||
545 | // In the case of Rc<Self>, we need to explicitly pass a *mut RcBox<Self> | |
546 | // with a Scalar (not ScalarPair) ABI. This is a hack that is understood | |
547 | // elsewhere in the compiler as a method on a `dyn Trait`. | |
548 | // To get the type `*mut RcBox<Self>`, we just keep unwrapping newtypes until we | |
549 | // get a built-in pointer type | |
550 | let mut fat_pointer_layout = layout; | |
551 | 'descend_newtypes: while !fat_pointer_layout.ty.is_unsafe_ptr() | |
353b0b11 | 552 | && !fat_pointer_layout.ty.is_ref() |
2b03887a FG |
553 | { |
554 | for i in 0..fat_pointer_layout.fields.count() { | |
555 | let field_layout = fat_pointer_layout.field(cx, i); | |
556 | ||
557 | if !field_layout.is_zst() { | |
558 | fat_pointer_layout = field_layout; | |
559 | continue 'descend_newtypes; | |
560 | } | |
561 | } | |
562 | ||
563 | bug!("receiver has no non-zero-sized fields {:?}", fat_pointer_layout); | |
564 | } | |
565 | ||
566 | fat_pointer_layout.ty | |
567 | }; | |
568 | ||
569 | // we now have a type like `*mut RcBox<dyn Trait>` | |
570 | // change its layout to that of `*mut ()`, a thin pointer, but keep the same type | |
571 | // this is understood as a special case elsewhere in the compiler | |
572 | let unit_ptr_ty = tcx.mk_mut_ptr(tcx.mk_unit()); | |
573 | ||
574 | TyAndLayout { | |
575 | ty: fat_pointer_ty, | |
576 | ||
577 | // NOTE(eddyb) using an empty `ParamEnv`, and `unwrap`-ing the `Result` | |
578 | // should always work because the type is always `*mut ()`. | |
579 | ..tcx.layout_of(ty::ParamEnv::reveal_all().and(unit_ptr_ty)).unwrap() | |
580 | } | |
581 | } |