//! Inlining pass for MIR functions
+use rustc::hir;
use rustc::hir::def_id::DefId;
use rustc_data_structures::bitvec::BitVector;
use rustc_data_structures::indexed_vec::{Idx, IndexVec};
use rustc::mir::*;
-use rustc::mir::transform::{MirPass, MirSource};
use rustc::mir::visit::*;
-use rustc::ty::{self, Ty, TyCtxt, Instance};
+use rustc::ty::{self, Instance, Ty, TyCtxt, TypeFoldable};
use rustc::ty::subst::{Subst,Substs};
use std::collections::VecDeque;
+use transform::{MirPass, MirSource};
use super::simplify::{remove_dead_blocks, CfgSimplifier};
use syntax::{attr};
pub struct Inline;
-#[derive(Copy, Clone)]
+#[derive(Copy, Clone, Debug)]
struct CallSite<'tcx> {
callee: DefId,
substs: &'tcx Substs<'tcx>,
let mut callsites = VecDeque::new();
+ let param_env = self.tcx.param_env(self.source.def_id);
+
// Only do inlining into fn bodies.
- if let MirSource::Fn(caller_id) = self.source {
+ let id = self.tcx.hir.as_local_node_id(self.source.def_id).unwrap();
+ let body_owner_kind = self.tcx.hir.body_owner_kind(id);
+ if let (hir::BodyOwnerKind::Fn, None) = (body_owner_kind, self.source.promoted) {
+
for (bb, bb_data) in caller_mir.basic_blocks().iter_enumerated() {
// Don't inline calls that are in cleanup blocks.
if bb_data.is_cleanup { continue; }
if let TerminatorKind::Call {
func: Operand::Constant(ref f), .. } = terminator.kind {
if let ty::TyFnDef(callee_def_id, substs) = f.ty.sty {
- let caller_def_id = self.tcx.hir.local_def_id(caller_id);
- let param_env = self.tcx.param_env(caller_def_id);
-
if let Some(instance) = Instance::resolve(self.tcx,
param_env,
callee_def_id,
}
}
}
+ } else {
+ return;
}
let mut local_change;
loop {
local_change = false;
while let Some(callsite) = callsites.pop_front() {
+ debug!("checking whether to inline callsite {:?}", callsite);
if !self.tcx.is_mir_available(callsite.callee) {
+ debug!("checking whether to inline callsite {:?} - MIR unavailable", callsite);
continue;
}
callsite.location.span,
callsite.callee) {
Ok(ref callee_mir) if self.should_inline(callsite, callee_mir) => {
- callee_mir.subst(self.tcx, callsite.substs)
+ subst_and_normalize(callee_mir, self.tcx, &callsite.substs, param_env)
}
Ok(_) => continue,
};
let start = caller_mir.basic_blocks().len();
-
+ debug!("attempting to inline callsite {:?} - mir={:?}", callsite, callee_mir);
if !self.inline_call(callsite, caller_mir, callee_mir) {
+ debug!("attempting to inline callsite {:?} - failure", callsite);
continue;
}
+ debug!("attempting to inline callsite {:?} - success", callsite);
// Add callsites from inlined function
for (bb, bb_data) in caller_mir.basic_blocks().iter_enumerated().skip(start) {
callee_mir: &Mir<'tcx>)
-> bool
{
+ debug!("should_inline({:?})", callsite);
let tcx = self.tcx;
// Don't inline closures that have captures
// FIXME: Handle closures better
if callee_mir.upvar_decls.len() > 0 {
+ debug!(" upvar decls present - not inlining");
return false;
}
// Cannot inline generators which haven't been transformed yet
if callee_mir.yield_ty.is_some() {
+ debug!(" yield ty present - not inlining");
return false;
}
// there are cases that prevent inlining that we
// need to check for first.
attr::InlineAttr::Always => true,
- attr::InlineAttr::Never => return false,
+ attr::InlineAttr::Never => {
+ debug!("#[inline(never)] present - not inlining");
+ return false
+ }
attr::InlineAttr::Hint => true,
attr::InlineAttr::None => false,
};
// reference unexported symbols
if callsite.callee.is_local() {
if callsite.substs.types().count() == 0 && !hinted {
+ debug!(" callee is an exported function - not inlining");
return false;
}
}
if callee_mir.basic_blocks().len() <= 3 {
threshold += threshold / 4;
}
+ debug!(" final inline threshold = {}", threshold);
// FIXME: Give a bonus to functions with only a single caller
- let def_id = tcx.hir.local_def_id(self.source.item_id());
- let param_env = tcx.param_env(def_id);
+ let param_env = tcx.param_env(self.source.def_id);
let mut first_block = true;
let mut cost = 0;
}
}
- debug!("Inline cost for {:?} is {}", callsite.callee, cost);
-
if let attr::InlineAttr::Always = hint {
+ debug!("INLINING {:?} because inline(always) [cost={}]", callsite, cost);
true
} else {
- cost <= threshold
+ if cost <= threshold {
+ debug!("INLINING {:?} [cost={} <= threshold={}]", callsite, cost, threshold);
+ true
+ } else {
+ debug!("NOT inlining {:?} [cost={} > threshold={}]", callsite, cost, threshold);
+ false
+ }
}
}
Operand::Consume(cast_tmp)
}
- fn make_call_args(&self, args: Vec<Operand<'tcx>>,
- callsite: &CallSite<'tcx>, caller_mir: &mut Mir<'tcx>) -> Vec<Operand<'tcx>> {
+ fn make_call_args(
+ &self,
+ args: Vec<Operand<'tcx>>,
+ callsite: &CallSite<'tcx>,
+ caller_mir: &mut Mir<'tcx>,
+ ) -> Vec<Operand<'tcx>> {
let tcx = self.tcx;
+
+ // A closure is passed its self-type and a tuple like `(arg1, arg2, ...)`,
+ // hence mappings to tuple fields are needed.
+ if tcx.is_closure(callsite.callee) {
+ let mut args = args.into_iter();
+ let self_ = self.create_temp_if_necessary(args.next().unwrap(), callsite, caller_mir);
+ let tuple = self.create_temp_if_necessary(args.next().unwrap(), callsite, caller_mir);
+ assert!(args.next().is_none());
+
+ let tuple_tys = if let ty::TyTuple(s, _) = tuple.ty(caller_mir, tcx).to_ty(tcx).sty {
+ s
+ } else {
+ bug!("Closure arguments are not passed as a tuple");
+ };
+
+ let mut res = Vec::with_capacity(1 + tuple_tys.len());
+ res.push(Operand::Consume(self_));
+ res.extend(tuple_tys.iter().enumerate().map(|(i, ty)| {
+ Operand::Consume(tuple.clone().field(Field::new(i), ty))
+ }));
+ res
+ } else {
+ args.into_iter()
+ .map(|a| Operand::Consume(self.create_temp_if_necessary(a, callsite, caller_mir)))
+ .collect()
+ }
+ }
+
+ /// If `arg` is already a temporary, returns it. Otherwise, introduces a fresh
+ /// temporary `T` and an instruction `T = arg`, and returns `T`.
+ fn create_temp_if_necessary(
+ &self,
+ arg: Operand<'tcx>,
+ callsite: &CallSite<'tcx>,
+ caller_mir: &mut Mir<'tcx>,
+ ) -> Lvalue<'tcx> {
// FIXME: Analysis of the usage of the arguments to avoid
// unnecessary temporaries.
- args.into_iter().map(|a| {
- if let Operand::Consume(Lvalue::Local(local)) = a {
- if caller_mir.local_kind(local) == LocalKind::Temp {
- // Reuse the operand if it's a temporary already
- return a;
- }
+
+ if let Operand::Consume(Lvalue::Local(local)) = arg {
+ if caller_mir.local_kind(local) == LocalKind::Temp {
+ // Reuse the operand if it's a temporary already
+ return Lvalue::Local(local);
}
+ }
- debug!("Creating temp for argument");
- // Otherwise, create a temporary for the arg
- let arg = Rvalue::Use(a);
+ debug!("Creating temp for argument {:?}", arg);
+ // Otherwise, create a temporary for the arg
+ let arg = Rvalue::Use(arg);
- let ty = arg.ty(caller_mir, tcx);
+ let ty = arg.ty(caller_mir, self.tcx);
- let arg_tmp = LocalDecl::new_temp(ty, callsite.location.span);
- let arg_tmp = caller_mir.local_decls.push(arg_tmp);
- let arg_tmp = Lvalue::Local(arg_tmp);
+ let arg_tmp = LocalDecl::new_temp(ty, callsite.location.span);
+ let arg_tmp = caller_mir.local_decls.push(arg_tmp);
+ let arg_tmp = Lvalue::Local(arg_tmp);
- let stmt = Statement {
- source_info: callsite.location,
- kind: StatementKind::Assign(arg_tmp.clone(), arg)
- };
- caller_mir[callsite.bb].statements.push(stmt);
- Operand::Consume(arg_tmp)
- }).collect()
+ let stmt = Statement {
+ source_info: callsite.location,
+ kind: StatementKind::Assign(arg_tmp.clone(), arg),
+ };
+ caller_mir[callsite.bb].statements.push(stmt);
+ arg_tmp
}
}
})
}
+fn subst_and_normalize<'a, 'tcx: 'a>(
+ mir: &Mir<'tcx>,
+ tcx: TyCtxt<'a, 'tcx, 'tcx>,
+ substs: &'tcx ty::subst::Substs<'tcx>,
+ param_env: ty::ParamEnv<'tcx>,
+) -> Mir<'tcx> {
+ struct Folder<'a, 'tcx: 'a> {
+ tcx: TyCtxt<'a, 'tcx, 'tcx>,
+ param_env: ty::ParamEnv<'tcx>,
+ substs: &'tcx ty::subst::Substs<'tcx>,
+ }
+ impl<'a, 'tcx: 'a> ty::fold::TypeFolder<'tcx, 'tcx> for Folder<'a, 'tcx> {
+ fn tcx<'b>(&'b self) -> TyCtxt<'b, 'tcx, 'tcx> {
+ self.tcx
+ }
+
+ fn fold_ty(&mut self, t: Ty<'tcx>) -> Ty<'tcx> {
+ self.tcx.trans_apply_param_substs_env(&self.substs, self.param_env, &t)
+ }
+ }
+ let mut f = Folder { tcx, param_env, substs };
+ mir.fold_with(&mut f)
+}
+
/**
* Integrator.
*
}
}
TerminatorKind::Unreachable => { }
+ TerminatorKind::FalseEdges { ref mut real_target, ref mut imaginary_targets } => {
+ *real_target = self.update_target(*real_target);
+ for target in imaginary_targets {
+ *target = self.update_target(*target);
+ }
+ }
}
}
projects / rustc.git / blobdiff