compiler/rustc_mir_transform/src/shim.rs

   1 use rustc_hir as hir;
   2 use rustc_hir::def_id::DefId;
   3 use rustc_hir::lang_items::LangItem;
   4 use rustc_middle::mir::*;
   5 use rustc_middle::ty::query::Providers;
   6 use rustc_middle::ty::InternalSubsts;
   7 use rustc_middle::ty::{self, EarlyBinder, GeneratorSubsts, Ty, TyCtxt};
   8 use rustc_target::abi::VariantIdx;
   9
  10 use rustc_index::vec::{Idx, IndexVec};
  11
  12 use rustc_span::Span;
  13 use rustc_target::spec::abi::Abi;
  14
  15 use std::fmt;
  16 use std::iter;
  17
  18 use crate::util::expand_aggregate;
  19 use crate::{
  20     abort_unwinding_calls, add_call_guards, add_moves_for_packed_drops, deref_separator,
  21     pass_manager as pm, remove_noop_landing_pads, simplify,
  22 };
  23 use rustc_middle::mir::patch::MirPatch;
  24 use rustc_mir_dataflow::elaborate_drops::{self, DropElaborator, DropFlagMode, DropStyle};
  25
  26 pub fn provide(providers: &mut Providers) {
  27     providers.mir_shims = make_shim;
  28 }
  29
  30 fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceDef<'tcx>) -> Body<'tcx> {
  31     debug!("make_shim({:?})", instance);
  32
  33     let mut result = match instance {
  34         ty::InstanceDef::Item(..) => bug!("item {:?} passed to make_shim", instance),
  35         ty::InstanceDef::VTableShim(def_id) => {
  36             build_call_shim(tcx, instance, Some(Adjustment::Deref), CallKind::Direct(def_id))
  37         }
  38         ty::InstanceDef::FnPtrShim(def_id, ty) => {
  39             let trait_ = tcx.trait_of_item(def_id).unwrap();
  40             let adjustment = match tcx.fn_trait_kind_from_def_id(trait_) {
  41                 Some(ty::ClosureKind::FnOnce) => Adjustment::Identity,
  42                 Some(ty::ClosureKind::FnMut | ty::ClosureKind::Fn) => Adjustment::Deref,
  43                 None => bug!("fn pointer {:?} is not an fn", ty),
  44             };
  45
  46             build_call_shim(tcx, instance, Some(adjustment), CallKind::Indirect(ty))
  47         }
  48         // We are generating a call back to our def-id, which the
  49         // codegen backend knows to turn to an actual call, be it
  50         // a virtual call, or a direct call to a function for which
  51         // indirect calls must be codegen'd differently than direct ones
  52         // (such as `#[track_caller]`).
  53         ty::InstanceDef::ReifyShim(def_id) => {
  54             build_call_shim(tcx, instance, None, CallKind::Direct(def_id))
  55         }
  56         ty::InstanceDef::ClosureOnceShim { call_once: _, track_caller: _ } => {
  57             let fn_mut = tcx.require_lang_item(LangItem::FnMut, None);
  58             let call_mut = tcx
  59                 .associated_items(fn_mut)
  60                 .in_definition_order()
  61                 .find(|it| it.kind == ty::AssocKind::Fn)
  62                 .unwrap()
  63                 .def_id;
  64
  65             build_call_shim(tcx, instance, Some(Adjustment::RefMut), CallKind::Direct(call_mut))
  66         }
  67
  68         ty::InstanceDef::DropGlue(def_id, ty) => {
  69             // FIXME(#91576): Drop shims for generators aren't subject to the MIR passes at the end
  70             // of this function. Is this intentional?
  71             if let Some(ty::Generator(gen_def_id, substs, _)) = ty.map(Ty::kind) {
  72                 let body = tcx.optimized_mir(*gen_def_id).generator_drop().unwrap();
  73                 let body = EarlyBinder(body.clone()).subst(tcx, substs);
  74                 debug!("make_shim({:?}) = {:?}", instance, body);
  75                 return body;
  76             }
  77
  78             build_drop_shim(tcx, def_id, ty)
  79         }
  80         ty::InstanceDef::CloneShim(def_id, ty) => build_clone_shim(tcx, def_id, ty),
  81         ty::InstanceDef::Virtual(..) => {
  82             bug!("InstanceDef::Virtual ({:?}) is for direct calls only", instance)
  83         }
  84         ty::InstanceDef::Intrinsic(_) => {
  85             bug!("creating shims from intrinsics ({:?}) is unsupported", instance)
  86         }
  87     };
  88     debug!("make_shim({:?}) = untransformed {:?}", instance, result);
  89
  90     pm::run_passes(
  91         tcx,
  92         &mut result,
  93         &[
  94             &add_moves_for_packed_drops::AddMovesForPackedDrops,
  95             &deref_separator::Derefer,
  96             &remove_noop_landing_pads::RemoveNoopLandingPads,
  97             &simplify::SimplifyCfg::new("make_shim"),
  98             &add_call_guards::CriticalCallEdges,
  99             &abort_unwinding_calls::AbortUnwindingCalls,
 100         ],
 101         Some(MirPhase::Runtime(RuntimePhase::Optimized)),
 102     );
 103
 104     debug!("make_shim({:?}) = {:?}", instance, result);
 105
 106     result
 107 }
 108
 109 #[derive(Copy, Clone, Debug, PartialEq)]
 110 enum Adjustment {
 111     /// Pass the receiver as-is.
 112     Identity,
 113
 114     /// We get passed `&[mut] self` and call the target with `*self`.
 115     ///
 116     /// This either copies `self` (if `Self: Copy`, eg. for function items), or moves out of it
 117     /// (for `VTableShim`, which effectively is passed `&own Self`).
 118     Deref,
 119
 120     /// We get passed `self: Self` and call the target with `&mut self`.
 121     ///
 122     /// In this case we need to ensure that the `Self` is dropped after the call, as the callee
 123     /// won't do it for us.
 124     RefMut,
 125 }
 126
 127 #[derive(Copy, Clone, Debug, PartialEq)]
 128 enum CallKind<'tcx> {
 129     /// Call the `FnPtr` that was passed as the receiver.
 130     Indirect(Ty<'tcx>),
 131
 132     /// Call a known `FnDef`.
 133     Direct(DefId),
 134 }
 135
 136 fn local_decls_for_sig<'tcx>(
 137     sig: &ty::FnSig<'tcx>,
 138     span: Span,
 139 ) -> IndexVec<Local, LocalDecl<'tcx>> {
 140     iter::once(LocalDecl::new(sig.output(), span))
 141         .chain(sig.inputs().iter().map(|ity| LocalDecl::new(*ity, span).immutable()))
 142         .collect()
 143 }
 144
 145 fn build_drop_shim<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, ty: Option<Ty<'tcx>>) -> Body<'tcx> {
 146     debug!("build_drop_shim(def_id={:?}, ty={:?})", def_id, ty);
 147
 148     assert!(!matches!(ty, Some(ty) if ty.is_generator()));
 149
 150     let substs = if let Some(ty) = ty {
 151         tcx.intern_substs(&[ty.into()])
 152     } else {
 153         InternalSubsts::identity_for_item(tcx, def_id)
 154     };
 155     let sig = tcx.bound_fn_sig(def_id).subst(tcx, substs);
 156     let sig = tcx.erase_late_bound_regions(sig);
 157     let span = tcx.def_span(def_id);
 158
 159     let source_info = SourceInfo::outermost(span);
 160
 161     let return_block = BasicBlock::new(1);
 162     let mut blocks = IndexVec::with_capacity(2);
 163     let block = |blocks: &mut IndexVec<_, _>, kind| {
 164         blocks.push(BasicBlockData {
 165             statements: vec![],
 166             terminator: Some(Terminator { source_info, kind }),
 167             is_cleanup: false,
 168         })
 169     };
 170     block(&mut blocks, TerminatorKind::Goto { target: return_block });
 171     block(&mut blocks, TerminatorKind::Return);
 172
 173     let source = MirSource::from_instance(ty::InstanceDef::DropGlue(def_id, ty));
 174     let mut body =
 175         new_body(source, blocks, local_decls_for_sig(&sig, span), sig.inputs().len(), span);
 176
 177     // The first argument (index 0), but add 1 for the return value.
 178     let mut dropee_ptr = Place::from(Local::new(1 + 0));
 179     if tcx.sess.opts.unstable_opts.mir_emit_retag {
 180         // We want to treat the function argument as if it was passed by `&mut`. As such, we
 181         // generate
 182         // ```
 183         // temp = &mut *arg;
 184         // Retag(temp, FnEntry)
 185         // ```
 186         // It's important that we do this first, before anything that depends on `dropee_ptr`
 187         // has been put into the body.
 188         let reborrow = Rvalue::Ref(
 189             tcx.lifetimes.re_erased,
 190             BorrowKind::Mut { allow_two_phase_borrow: false },
 191             tcx.mk_place_deref(dropee_ptr),
 192         );
 193         let ref_ty = reborrow.ty(body.local_decls(), tcx);
 194         dropee_ptr = body.local_decls.push(LocalDecl::new(ref_ty, span)).into();
 195         let new_statements = [
 196             StatementKind::Assign(Box::new((dropee_ptr, reborrow))),
 197             StatementKind::Retag(RetagKind::FnEntry, Box::new(dropee_ptr)),
 198         ];
 199         for s in new_statements {
 200             body.basic_blocks_mut()[START_BLOCK]
 201                 .statements
 202                 .push(Statement { source_info, kind: s });
 203         }
 204     }
 205
 206     if ty.is_some() {
 207         let patch = {
 208             let param_env = tcx.param_env_reveal_all_normalized(def_id);
 209             let mut elaborator =
 210                 DropShimElaborator { body: &body, patch: MirPatch::new(&body), tcx, param_env };
 211             let dropee = tcx.mk_place_deref(dropee_ptr);
 212             let resume_block = elaborator.patch.resume_block();
 213             elaborate_drops::elaborate_drop(
 214                 &mut elaborator,
 215                 source_info,
 216                 dropee,
 217                 (),
 218                 return_block,
 219                 elaborate_drops::Unwind::To(resume_block),
 220                 START_BLOCK,
 221             );
 222             elaborator.patch
 223         };
 224         patch.apply(&mut body);
 225     }
 226
 227     body
 228 }
 229
 230 fn new_body<'tcx>(
 231     source: MirSource<'tcx>,
 232     basic_blocks: IndexVec<BasicBlock, BasicBlockData<'tcx>>,
 233     local_decls: IndexVec<Local, LocalDecl<'tcx>>,
 234     arg_count: usize,
 235     span: Span,
 236 ) -> Body<'tcx> {
 237     Body::new(
 238         source,
 239         basic_blocks,
 240         IndexVec::from_elem_n(
 241             SourceScopeData {
 242                 span,
 243                 parent_scope: None,
 244                 inlined: None,
 245                 inlined_parent_scope: None,
 246                 local_data: ClearCrossCrate::Clear,
 247             },
 248             1,
 249         ),
 250         local_decls,
 251         IndexVec::new(),
 252         arg_count,
 253         vec![],
 254         span,
 255         None,
 256         // FIXME(compiler-errors): is this correct?
 257         None,
 258     )
 259 }
 260
 261 pub struct DropShimElaborator<'a, 'tcx> {
 262     pub body: &'a Body<'tcx>,
 263     pub patch: MirPatch<'tcx>,
 264     pub tcx: TyCtxt<'tcx>,
 265     pub param_env: ty::ParamEnv<'tcx>,
 266 }
 267
 268 impl fmt::Debug for DropShimElaborator<'_, '_> {
 269     fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
 270         Ok(())
 271     }
 272 }
 273
 274 impl<'a, 'tcx> DropElaborator<'a, 'tcx> for DropShimElaborator<'a, 'tcx> {
 275     type Path = ();
 276
 277     fn patch(&mut self) -> &mut MirPatch<'tcx> {
 278         &mut self.patch
 279     }
 280     fn body(&self) -> &'a Body<'tcx> {
 281         self.body
 282     }
 283     fn tcx(&self) -> TyCtxt<'tcx> {
 284         self.tcx
 285     }
 286     fn param_env(&self) -> ty::ParamEnv<'tcx> {
 287         self.param_env
 288     }
 289
 290     fn drop_style(&self, _path: Self::Path, mode: DropFlagMode) -> DropStyle {
 291         match mode {
 292             DropFlagMode::Shallow => {
 293                 // Drops for the contained fields are "shallow" and "static" - they will simply call
 294                 // the field's own drop glue.
 295                 DropStyle::Static
 296             }
 297             DropFlagMode::Deep => {
 298                 // The top-level drop is "deep" and "open" - it will be elaborated to a drop ladder
 299                 // dropping each field contained in the value.
 300                 DropStyle::Open
 301             }
 302         }
 303     }
 304
 305     fn get_drop_flag(&mut self, _path: Self::Path) -> Option<Operand<'tcx>> {
 306         None
 307     }
 308
 309     fn clear_drop_flag(&mut self, _location: Location, _path: Self::Path, _mode: DropFlagMode) {}
 310
 311     fn field_subpath(&self, _path: Self::Path, _field: Field) -> Option<Self::Path> {
 312         None
 313     }
 314     fn deref_subpath(&self, _path: Self::Path) -> Option<Self::Path> {
 315         None
 316     }
 317     fn downcast_subpath(&self, _path: Self::Path, _variant: VariantIdx) -> Option<Self::Path> {
 318         Some(())
 319     }
 320     fn array_subpath(&self, _path: Self::Path, _index: u64, _size: u64) -> Option<Self::Path> {
 321         None
 322     }
 323 }
 324
 325 /// Builds a `Clone::clone` shim for `self_ty`. Here, `def_id` is `Clone::clone`.
 326 fn build_clone_shim<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, self_ty: Ty<'tcx>) -> Body<'tcx> {
 327     debug!("build_clone_shim(def_id={:?})", def_id);
 328
 329     let param_env = tcx.param_env(def_id);
 330
 331     let mut builder = CloneShimBuilder::new(tcx, def_id, self_ty);
 332     let is_copy = self_ty.is_copy_modulo_regions(tcx, param_env);
 333
 334     let dest = Place::return_place();
 335     let src = tcx.mk_place_deref(Place::from(Local::new(1 + 0)));
 336
 337     match self_ty.kind() {
 338         _ if is_copy => builder.copy_shim(),
 339         ty::Closure(_, substs) => {
 340             builder.tuple_like_shim(dest, src, substs.as_closure().upvar_tys())
 341         }
 342         ty::Tuple(..) => builder.tuple_like_shim(dest, src, self_ty.tuple_fields()),
 343         ty::Generator(gen_def_id, substs, hir::Movability::Movable) => {
 344             builder.generator_shim(dest, src, *gen_def_id, substs.as_generator())
 345         }
 346         _ => bug!("clone shim for `{:?}` which is not `Copy` and is not an aggregate", self_ty),
 347     };
 348
 349     builder.into_mir()
 350 }
 351
 352 struct CloneShimBuilder<'tcx> {
 353     tcx: TyCtxt<'tcx>,
 354     def_id: DefId,
 355     local_decls: IndexVec<Local, LocalDecl<'tcx>>,
 356     blocks: IndexVec<BasicBlock, BasicBlockData<'tcx>>,
 357     span: Span,
 358     sig: ty::FnSig<'tcx>,
 359 }
 360
 361 impl<'tcx> CloneShimBuilder<'tcx> {
 362     fn new(tcx: TyCtxt<'tcx>, def_id: DefId, self_ty: Ty<'tcx>) -> Self {
 363         // we must subst the self_ty because it's
 364         // otherwise going to be TySelf and we can't index
 365         // or access fields of a Place of type TySelf.
 366         let sig = tcx.bound_fn_sig(def_id).subst(tcx, &[self_ty.into()]);
 367         let sig = tcx.erase_late_bound_regions(sig);
 368         let span = tcx.def_span(def_id);
 369
 370         CloneShimBuilder {
 371             tcx,
 372             def_id,
 373             local_decls: local_decls_for_sig(&sig, span),
 374             blocks: IndexVec::new(),
 375             span,
 376             sig,
 377         }
 378     }
 379
 380     fn into_mir(self) -> Body<'tcx> {
 381         let source = MirSource::from_instance(ty::InstanceDef::CloneShim(
 382             self.def_id,
 383             self.sig.inputs_and_output[0],
 384         ));
 385         new_body(source, self.blocks, self.local_decls, self.sig.inputs().len(), self.span)
 386     }
 387
 388     fn source_info(&self) -> SourceInfo {
 389         SourceInfo::outermost(self.span)
 390     }
 391
 392     fn block(
 393         &mut self,
 394         statements: Vec<Statement<'tcx>>,
 395         kind: TerminatorKind<'tcx>,
 396         is_cleanup: bool,
 397     ) -> BasicBlock {
 398         let source_info = self.source_info();
 399         self.blocks.push(BasicBlockData {
 400             statements,
 401             terminator: Some(Terminator { source_info, kind }),
 402             is_cleanup,
 403         })
 404     }
 405
 406     /// Gives the index of an upcoming BasicBlock, with an offset.
 407     /// offset=0 will give you the index of the next BasicBlock,
 408     /// offset=1 will give the index of the next-to-next block,
 409     /// offset=-1 will give you the index of the last-created block
 410     fn block_index_offset(&self, offset: usize) -> BasicBlock {
 411         BasicBlock::new(self.blocks.len() + offset)
 412     }
 413
 414     fn make_statement(&self, kind: StatementKind<'tcx>) -> Statement<'tcx> {
 415         Statement { source_info: self.source_info(), kind }
 416     }
 417
 418     fn copy_shim(&mut self) {
 419         let rcvr = self.tcx.mk_place_deref(Place::from(Local::new(1 + 0)));
 420         let ret_statement = self.make_statement(StatementKind::Assign(Box::new((
 421             Place::return_place(),
 422             Rvalue::Use(Operand::Copy(rcvr)),
 423         ))));
 424         self.block(vec![ret_statement], TerminatorKind::Return, false);
 425     }
 426
 427     fn make_place(&mut self, mutability: Mutability, ty: Ty<'tcx>) -> Place<'tcx> {
 428         let span = self.span;
 429         let mut local = LocalDecl::new(ty, span);
 430         if mutability == Mutability::Not {
 431             local = local.immutable();
 432         }
 433         Place::from(self.local_decls.push(local))
 434     }
 435
 436     fn make_clone_call(
 437         &mut self,
 438         dest: Place<'tcx>,
 439         src: Place<'tcx>,
 440         ty: Ty<'tcx>,
 441         next: BasicBlock,
 442         cleanup: BasicBlock,
 443     ) {
 444         let tcx = self.tcx;
 445
 446         // `func == Clone::clone(&ty) -> ty`
 447         let func_ty = tcx.mk_fn_def(self.def_id, [ty]);
 448         let func = Operand::Constant(Box::new(Constant {
 449             span: self.span,
 450             user_ty: None,
 451             literal: ConstantKind::zero_sized(func_ty),
 452         }));
 453
 454         let ref_loc = self.make_place(
 455             Mutability::Not,
 456             tcx.mk_ref(tcx.lifetimes.re_erased, ty::TypeAndMut { ty, mutbl: hir::Mutability::Not }),
 457         );
 458
 459         // `let ref_loc: &ty = &src;`
 460         let statement = self.make_statement(StatementKind::Assign(Box::new((
 461             ref_loc,
 462             Rvalue::Ref(tcx.lifetimes.re_erased, BorrowKind::Shared, src),
 463         ))));
 464
 465         // `let loc = Clone::clone(ref_loc);`
 466         self.block(
 467             vec![statement],
 468             TerminatorKind::Call {
 469                 func,
 470                 args: vec![Operand::Move(ref_loc)],
 471                 destination: dest,
 472                 target: Some(next),
 473                 cleanup: Some(cleanup),
 474                 from_hir_call: true,
 475                 fn_span: self.span,
 476             },
 477             false,
 478         );
 479     }
 480
 481     fn clone_fields<I>(
 482         &mut self,
 483         dest: Place<'tcx>,
 484         src: Place<'tcx>,
 485         target: BasicBlock,
 486         mut unwind: BasicBlock,
 487         tys: I,
 488     ) -> BasicBlock
 489     where
 490         I: IntoIterator<Item = Ty<'tcx>>,
 491     {
 492         // For an iterator of length n, create 2*n + 1 blocks.
 493         for (i, ity) in tys.into_iter().enumerate() {
 494             // Each iteration creates two blocks, referred to here as block 2*i and block 2*i + 1.
 495             //
 496             // Block 2*i attempts to clone the field. If successful it branches to 2*i + 2 (the
 497             // next clone block). If unsuccessful it branches to the previous unwind block, which
 498             // is initially the `unwind` argument passed to this function.
 499             //
 500             // Block 2*i + 1 is the unwind block for this iteration. It drops the cloned value
 501             // created by block 2*i. We store this block in `unwind` so that the next clone block
 502             // will unwind to it if cloning fails.
 503
 504             let field = Field::new(i);
 505             let src_field = self.tcx.mk_place_field(src, field, ity);
 506
 507             let dest_field = self.tcx.mk_place_field(dest, field, ity);
 508
 509             let next_unwind = self.block_index_offset(1);
 510             let next_block = self.block_index_offset(2);
 511             self.make_clone_call(dest_field, src_field, ity, next_block, unwind);
 512             self.block(
 513                 vec![],
 514                 TerminatorKind::Drop { place: dest_field, target: unwind, unwind: None },
 515                 true,
 516             );
 517             unwind = next_unwind;
 518         }
 519         // If all clones succeed then we end up here.
 520         self.block(vec![], TerminatorKind::Goto { target }, false);
 521         unwind
 522     }
 523
 524     fn tuple_like_shim<I>(&mut self, dest: Place<'tcx>, src: Place<'tcx>, tys: I)
 525     where
 526         I: IntoIterator<Item = Ty<'tcx>>,
 527     {
 528         self.block(vec![], TerminatorKind::Goto { target: self.block_index_offset(3) }, false);
 529         let unwind = self.block(vec![], TerminatorKind::Resume, true);
 530         let target = self.block(vec![], TerminatorKind::Return, false);
 531
 532         let _final_cleanup_block = self.clone_fields(dest, src, target, unwind, tys);
 533     }
 534
 535     fn generator_shim(
 536         &mut self,
 537         dest: Place<'tcx>,
 538         src: Place<'tcx>,
 539         gen_def_id: DefId,
 540         substs: GeneratorSubsts<'tcx>,
 541     ) {
 542         self.block(vec![], TerminatorKind::Goto { target: self.block_index_offset(3) }, false);
 543         let unwind = self.block(vec![], TerminatorKind::Resume, true);
 544         // This will get overwritten with a switch once we know the target blocks
 545         let switch = self.block(vec![], TerminatorKind::Unreachable, false);
 546         let unwind = self.clone_fields(dest, src, switch, unwind, substs.upvar_tys());
 547         let target = self.block(vec![], TerminatorKind::Return, false);
 548         let unreachable = self.block(vec![], TerminatorKind::Unreachable, false);
 549         let mut cases = Vec::with_capacity(substs.state_tys(gen_def_id, self.tcx).count());
 550         for (index, state_tys) in substs.state_tys(gen_def_id, self.tcx).enumerate() {
 551             let variant_index = VariantIdx::new(index);
 552             let dest = self.tcx.mk_place_downcast_unnamed(dest, variant_index);
 553             let src = self.tcx.mk_place_downcast_unnamed(src, variant_index);
 554             let clone_block = self.block_index_offset(1);
 555             let start_block = self.block(
 556                 vec![self.make_statement(StatementKind::SetDiscriminant {
 557                     place: Box::new(Place::return_place()),
 558                     variant_index,
 559                 })],
 560                 TerminatorKind::Goto { target: clone_block },
 561                 false,
 562             );
 563             cases.push((index as u128, start_block));
 564             let _final_cleanup_block = self.clone_fields(dest, src, target, unwind, state_tys);
 565         }
 566         let discr_ty = substs.discr_ty(self.tcx);
 567         let temp = self.make_place(Mutability::Mut, discr_ty);
 568         let rvalue = Rvalue::Discriminant(src);
 569         let statement = self.make_statement(StatementKind::Assign(Box::new((temp, rvalue))));
 570         match &mut self.blocks[switch] {
 571             BasicBlockData { statements, terminator: Some(Terminator { kind, .. }), .. } => {
 572                 statements.push(statement);
 573                 *kind = TerminatorKind::SwitchInt {
 574                     discr: Operand::Move(temp),
 575                     targets: SwitchTargets::new(cases.into_iter(), unreachable),
 576                 };
 577             }
 578             BasicBlockData { terminator: None, .. } => unreachable!(),
 579         }
 580     }
 581 }
 582
 583 /// Builds a "call" shim for `instance`. The shim calls the function specified by `call_kind`,
 584 /// first adjusting its first argument according to `rcvr_adjustment`.
 585 #[instrument(level = "debug", skip(tcx), ret)]
 586 fn build_call_shim<'tcx>(
 587     tcx: TyCtxt<'tcx>,
 588     instance: ty::InstanceDef<'tcx>,
 589     rcvr_adjustment: Option<Adjustment>,
 590     call_kind: CallKind<'tcx>,
 591 ) -> Body<'tcx> {
 592     // `FnPtrShim` contains the fn pointer type that a call shim is being built for - this is used
 593     // to substitute into the signature of the shim. It is not necessary for users of this
 594     // MIR body to perform further substitutions (see `InstanceDef::has_polymorphic_mir_body`).
 595     let (sig_substs, untuple_args) = if let ty::InstanceDef::FnPtrShim(_, ty) = instance {
 596         let sig = tcx.erase_late_bound_regions(ty.fn_sig(tcx));
 597
 598         let untuple_args = sig.inputs();
 599
 600         // Create substitutions for the `Self` and `Args` generic parameters of the shim body.
 601         let arg_tup = tcx.mk_tup(untuple_args.iter());
 602
 603         (Some([ty.into(), arg_tup.into()]), Some(untuple_args))
 604     } else {
 605         (None, None)
 606     };
 607
 608     let def_id = instance.def_id();
 609     let sig = tcx.bound_fn_sig(def_id);
 610     let sig = sig.map_bound(|sig| tcx.erase_late_bound_regions(sig));
 611
 612     assert_eq!(sig_substs.is_some(), !instance.has_polymorphic_mir_body());
 613     let mut sig =
 614         if let Some(sig_substs) = sig_substs { sig.subst(tcx, &sig_substs) } else { sig.0 };
 615
 616     if let CallKind::Indirect(fnty) = call_kind {
 617         // `sig` determines our local decls, and thus the callee type in the `Call` terminator. This
 618         // can only be an `FnDef` or `FnPtr`, but currently will be `Self` since the types come from
 619         // the implemented `FnX` trait.
 620
 621         // Apply the opposite adjustment to the MIR input.
 622         let mut inputs_and_output = sig.inputs_and_output.to_vec();
 623
 624         // Initial signature is `fn(&? Self, Args) -> Self::Output` where `Args` is a tuple of the
 625         // fn arguments. `Self` may be passed via (im)mutable reference or by-value.
 626         assert_eq!(inputs_and_output.len(), 3);
 627
 628         // `Self` is always the original fn type `ty`. The MIR call terminator is only defined for
 629         // `FnDef` and `FnPtr` callees, not the `Self` type param.
 630         let self_arg = &mut inputs_and_output[0];
 631         *self_arg = match rcvr_adjustment.unwrap() {
 632             Adjustment::Identity => fnty,
 633             Adjustment::Deref => tcx.mk_imm_ptr(fnty),
 634             Adjustment::RefMut => tcx.mk_mut_ptr(fnty),
 635         };
 636         sig.inputs_and_output = tcx.intern_type_list(&inputs_and_output);
 637     }
 638
 639     // FIXME(eddyb) avoid having this snippet both here and in
 640     // `Instance::fn_sig` (introduce `InstanceDef::fn_sig`?).
 641     if let ty::InstanceDef::VTableShim(..) = instance {
 642         // Modify fn(self, ...) to fn(self: *mut Self, ...)
 643         let mut inputs_and_output = sig.inputs_and_output.to_vec();
 644         let self_arg = &mut inputs_and_output[0];
 645         debug_assert!(tcx.generics_of(def_id).has_self && *self_arg == tcx.types.self_param);
 646         *self_arg = tcx.mk_mut_ptr(*self_arg);
 647         sig.inputs_and_output = tcx.intern_type_list(&inputs_and_output);
 648     }
 649
 650     let span = tcx.def_span(def_id);
 651
 652     debug!(?sig);
 653
 654     let mut local_decls = local_decls_for_sig(&sig, span);
 655     let source_info = SourceInfo::outermost(span);
 656
 657     let rcvr_place = || {
 658         assert!(rcvr_adjustment.is_some());
 659         Place::from(Local::new(1 + 0))
 660     };
 661     let mut statements = vec![];
 662
 663     let rcvr = rcvr_adjustment.map(|rcvr_adjustment| match rcvr_adjustment {
 664         Adjustment::Identity => Operand::Move(rcvr_place()),
 665         Adjustment::Deref => Operand::Move(tcx.mk_place_deref(rcvr_place())),
 666         Adjustment::RefMut => {
 667             // let rcvr = &mut rcvr;
 668             let ref_rcvr = local_decls.push(
 669                 LocalDecl::new(
 670                     tcx.mk_ref(
 671                         tcx.lifetimes.re_erased,
 672                         ty::TypeAndMut { ty: sig.inputs()[0], mutbl: hir::Mutability::Mut },
 673                     ),
 674                     span,
 675                 )
 676                 .immutable(),
 677             );
 678             let borrow_kind = BorrowKind::Mut { allow_two_phase_borrow: false };
 679             statements.push(Statement {
 680                 source_info,
 681                 kind: StatementKind::Assign(Box::new((
 682                     Place::from(ref_rcvr),
 683                     Rvalue::Ref(tcx.lifetimes.re_erased, borrow_kind, rcvr_place()),
 684                 ))),
 685             });
 686             Operand::Move(Place::from(ref_rcvr))
 687         }
 688     });
 689
 690     let (callee, mut args) = match call_kind {
 691         // `FnPtr` call has no receiver. Args are untupled below.
 692         CallKind::Indirect(_) => (rcvr.unwrap(), vec![]),
 693
 694         // `FnDef` call with optional receiver.
 695         CallKind::Direct(def_id) => {
 696             let ty = tcx.type_of(def_id);
 697             (
 698                 Operand::Constant(Box::new(Constant {
 699                     span,
 700                     user_ty: None,
 701                     literal: ConstantKind::zero_sized(ty),
 702                 })),
 703                 rcvr.into_iter().collect::<Vec<_>>(),
 704             )
 705         }
 706     };
 707
 708     let mut arg_range = 0..sig.inputs().len();
 709
 710     // Take the `self` ("receiver") argument out of the range (it's adjusted above).
 711     if rcvr_adjustment.is_some() {
 712         arg_range.start += 1;
 713     }
 714
 715     // Take the last argument, if we need to untuple it (handled below).
 716     if untuple_args.is_some() {
 717         arg_range.end -= 1;
 718     }
 719
 720     // Pass all of the non-special arguments directly.
 721     args.extend(arg_range.map(|i| Operand::Move(Place::from(Local::new(1 + i)))));
 722
 723     // Untuple the last argument, if we have to.
 724     if let Some(untuple_args) = untuple_args {
 725         let tuple_arg = Local::new(1 + (sig.inputs().len() - 1));
 726         args.extend(untuple_args.iter().enumerate().map(|(i, ity)| {
 727             Operand::Move(tcx.mk_place_field(Place::from(tuple_arg), Field::new(i), *ity))
 728         }));
 729     }
 730
 731     let n_blocks = if let Some(Adjustment::RefMut) = rcvr_adjustment { 5 } else { 2 };
 732     let mut blocks = IndexVec::with_capacity(n_blocks);
 733     let block = |blocks: &mut IndexVec<_, _>, statements, kind, is_cleanup| {
 734         blocks.push(BasicBlockData {
 735             statements,
 736             terminator: Some(Terminator { source_info, kind }),
 737             is_cleanup,
 738         })
 739     };
 740
 741     // BB #0
 742     block(
 743         &mut blocks,
 744         statements,
 745         TerminatorKind::Call {
 746             func: callee,
 747             args,
 748             destination: Place::return_place(),
 749             target: Some(BasicBlock::new(1)),
 750             cleanup: if let Some(Adjustment::RefMut) = rcvr_adjustment {
 751                 Some(BasicBlock::new(3))
 752             } else {
 753                 None
 754             },
 755             from_hir_call: true,
 756             fn_span: span,
 757         },
 758         false,
 759     );
 760
 761     if let Some(Adjustment::RefMut) = rcvr_adjustment {
 762         // BB #1 - drop for Self
 763         block(
 764             &mut blocks,
 765             vec![],
 766             TerminatorKind::Drop { place: rcvr_place(), target: BasicBlock::new(2), unwind: None },
 767             false,
 768         );
 769     }
 770     // BB #1/#2 - return
 771     block(&mut blocks, vec![], TerminatorKind::Return, false);
 772     if let Some(Adjustment::RefMut) = rcvr_adjustment {
 773         // BB #3 - drop if closure panics
 774         block(
 775             &mut blocks,
 776             vec![],
 777             TerminatorKind::Drop { place: rcvr_place(), target: BasicBlock::new(4), unwind: None },
 778             true,
 779         );
 780
 781         // BB #4 - resume
 782         block(&mut blocks, vec![], TerminatorKind::Resume, true);
 783     }
 784
 785     let mut body =
 786         new_body(MirSource::from_instance(instance), blocks, local_decls, sig.inputs().len(), span);
 787
 788     if let Abi::RustCall = sig.abi {
 789         body.spread_arg = Some(Local::new(sig.inputs().len()));
 790     }
 791
 792     body
 793 }
 794
 795 pub fn build_adt_ctor(tcx: TyCtxt<'_>, ctor_id: DefId) -> Body<'_> {
 796     debug_assert!(tcx.is_constructor(ctor_id));
 797
 798     let param_env = tcx.param_env(ctor_id);
 799
 800     // Normalize the sig.
 801     let sig = tcx.fn_sig(ctor_id).no_bound_vars().expect("LBR in ADT constructor signature");
 802     let sig = tcx.normalize_erasing_regions(param_env, sig);
 803
 804     let ty::Adt(adt_def, substs) = sig.output().kind() else {
 805         bug!("unexpected type for ADT ctor {:?}", sig.output());
 806     };
 807
 808     debug!("build_ctor: ctor_id={:?} sig={:?}", ctor_id, sig);
 809
 810     let span = tcx.def_span(ctor_id);
 811
 812     let local_decls = local_decls_for_sig(&sig, span);
 813
 814     let source_info = SourceInfo::outermost(span);
 815
 816     let variant_index = if adt_def.is_enum() {
 817         adt_def.variant_index_with_ctor_id(ctor_id)
 818     } else {
 819         VariantIdx::new(0)
 820     };
 821
 822     // Generate the following MIR:
 823     //
 824     // (return as Variant).field0 = arg0;
 825     // (return as Variant).field1 = arg1;
 826     //
 827     // return;
 828     debug!("build_ctor: variant_index={:?}", variant_index);
 829
 830     let statements = expand_aggregate(
 831         Place::return_place(),
 832         adt_def.variant(variant_index).fields.iter().enumerate().map(|(idx, field_def)| {
 833             (Operand::Move(Place::from(Local::new(idx + 1))), field_def.ty(tcx, substs))
 834         }),
 835         AggregateKind::Adt(adt_def.did(), variant_index, substs, None, None),
 836         source_info,
 837         tcx,
 838     )
 839     .collect();
 840
 841     let start_block = BasicBlockData {
 842         statements,
 843         terminator: Some(Terminator { source_info, kind: TerminatorKind::Return }),
 844         is_cleanup: false,
 845     };
 846
 847     let source = MirSource::item(ctor_id);
 848     let body = new_body(
 849         source,
 850         IndexVec::from_elem_n(start_block, 1),
 851         local_decls,
 852         sig.inputs().len(),
 853         span,
 854     );
 855
 856     crate::pass_manager::dump_mir_for_phase_change(tcx, &body);
 857
 858     body
 859 }