1 //! A pass that promotes borrows of constant rvalues.
3 //! The rvalues considered constant are trees of temps,
4 //! each with exactly one initialization, and holding
5 //! a constant value with no interior mutability.
6 //! They are placed into a new MIR constant body in
7 //! `promoted` and the borrow rvalue is replaced with
8 //! a `Literal::Promoted` using the index into `promoted`
9 //! of that constant MIR.
11 //! This pass assumes that every use is dominated by an
12 //! initialization and can otherwise silence errors, if
13 //! move analysis runs after promotion on broken MIR.
16 use rustc_middle
::mir
::traversal
::ReversePostorderIter
;
17 use rustc_middle
::mir
::visit
::{MutVisitor, MutatingUseContext, PlaceContext, Visitor}
;
18 use rustc_middle
::mir
::*;
19 use rustc_middle
::ty
::subst
::InternalSubsts
;
20 use rustc_middle
::ty
::{self, List, TyCtxt, TypeFoldable}
;
23 use rustc_index
::vec
::{Idx, IndexVec}
;
26 use std
::{cmp, iter, mem}
;
28 use crate::transform
::check_consts
::{qualifs, ConstCx}
;
30 /// A `MirPass` for promotion.
32 /// Promotion is the extraction of promotable temps into separate MIR bodies so they can have
33 /// `'static` lifetime.
35 /// After this pass is run, `promoted_fragments` will hold the MIR body corresponding to each
36 /// newly created `Constant`.
38 pub struct PromoteTemps
<'tcx
> {
39 pub promoted_fragments
: Cell
<IndexVec
<Promoted
, Body
<'tcx
>>>,
42 impl<'tcx
> MirPass
<'tcx
> for PromoteTemps
<'tcx
> {
43 fn phase_change(&self) -> Option
<MirPhase
> {
44 Some(MirPhase
::ConstsPromoted
)
47 fn run_pass(&self, tcx
: TyCtxt
<'tcx
>, body
: &mut Body
<'tcx
>) {
48 // There's not really any point in promoting errorful MIR.
50 // This does not include MIR that failed const-checking, which we still try to promote.
51 if body
.return_ty().references_error() {
52 tcx
.sess
.delay_span_bug(body
.span
, "PromoteTemps: MIR had errors");
56 if body
.source
.promoted
.is_some() {
60 let mut rpo
= traversal
::reverse_postorder(body
);
61 let ccx
= ConstCx
::new(tcx
, body
);
62 let (mut temps
, all_candidates
) = collect_temps_and_candidates(&ccx
, &mut rpo
);
64 let promotable_candidates
= validate_candidates(&ccx
, &mut temps
, &all_candidates
);
66 let promoted
= promote_candidates(body
, tcx
, temps
, promotable_candidates
);
67 self.promoted_fragments
.set(promoted
);
71 /// State of a temporary during collection and promotion.
72 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
74 /// No references to this temp.
76 /// One direct assignment and any number of direct uses.
77 /// A borrow of this temp is promotable if the assigned
78 /// value is qualified as constant.
79 Defined { location: Location, uses: usize, valid: Result<(), ()> }
,
80 /// Any other combination of assignments/uses.
82 /// This temp was part of an rvalue which got extracted
83 /// during promotion and needs cleanup.
88 pub fn is_promotable(&self) -> bool
{
89 debug
!("is_promotable: self={:?}", self);
90 matches
!(self, TempState
::Defined { .. }
)
94 /// A "root candidate" for promotion, which will become the
95 /// returned value in a promoted MIR, unless it's a subset
96 /// of a larger candidate.
97 #[derive(Copy, Clone, PartialEq, Eq, Debug)]
98 pub struct Candidate
{
102 struct Collector
<'a
, 'tcx
> {
103 ccx
: &'a ConstCx
<'a
, 'tcx
>,
104 temps
: IndexVec
<Local
, TempState
>,
105 candidates
: Vec
<Candidate
>,
108 impl<'tcx
> Visitor
<'tcx
> for Collector
<'_
, 'tcx
> {
109 fn visit_local(&mut self, &index
: &Local
, context
: PlaceContext
, location
: Location
) {
110 debug
!("visit_local: index={:?} context={:?} location={:?}", index
, context
, location
);
111 // We're only interested in temporaries and the return place
112 match self.ccx
.body
.local_kind(index
) {
113 LocalKind
::Temp
| LocalKind
::ReturnPointer
=> {}
114 LocalKind
::Arg
| LocalKind
::Var
=> return,
117 // Ignore drops, if the temp gets promoted,
118 // then it's constant and thus drop is noop.
119 // Non-uses are also irrelevant.
120 if context
.is_drop() || !context
.is_use() {
122 "visit_local: context.is_drop={:?} context.is_use={:?}",
129 let temp
= &mut self.temps
[index
];
130 debug
!("visit_local: temp={:?}", temp
);
131 if *temp
== TempState
::Undefined
{
133 PlaceContext
::MutatingUse(MutatingUseContext
::Store
)
134 | PlaceContext
::MutatingUse(MutatingUseContext
::Call
) => {
135 *temp
= TempState
::Defined { location, uses: 0, valid: Err(()) }
;
138 _
=> { /* mark as unpromotable below */ }
140 } else if let TempState
::Defined { ref mut uses, .. }
= *temp
{
141 // We always allow borrows, even mutable ones, as we need
142 // to promote mutable borrows of some ZSTs e.g., `&mut []`.
143 let allowed_use
= match context
{
144 PlaceContext
::MutatingUse(MutatingUseContext
::Borrow
)
145 | PlaceContext
::NonMutatingUse(_
) => true,
146 PlaceContext
::MutatingUse(_
) | PlaceContext
::NonUse(_
) => false,
148 debug
!("visit_local: allowed_use={:?}", allowed_use
);
153 /* mark as unpromotable below */
155 *temp
= TempState
::Unpromotable
;
158 fn visit_rvalue(&mut self, rvalue
: &Rvalue
<'tcx
>, location
: Location
) {
159 self.super_rvalue(rvalue
, location
);
163 self.candidates
.push(Candidate { location }
);
170 pub fn collect_temps_and_candidates
<'tcx
>(
171 ccx
: &ConstCx
<'_
, 'tcx
>,
172 rpo
: &mut ReversePostorderIter
<'_
, 'tcx
>,
173 ) -> (IndexVec
<Local
, TempState
>, Vec
<Candidate
>) {
174 let mut collector
= Collector
{
175 temps
: IndexVec
::from_elem(TempState
::Undefined
, &ccx
.body
.local_decls
),
179 for (bb
, data
) in rpo
{
180 collector
.visit_basic_block_data(bb
, data
);
182 (collector
.temps
, collector
.candidates
)
185 /// Checks whether locals that appear in a promotion context (`Candidate`) are actually promotable.
187 /// This wraps an `Item`, and has access to all fields of that `Item` via `Deref` coercion.
188 struct Validator
<'a
, 'tcx
> {
189 ccx
: &'a ConstCx
<'a
, 'tcx
>,
190 temps
: &'a
mut IndexVec
<Local
, TempState
>,
193 impl<'a
, 'tcx
> std
::ops
::Deref
for Validator
<'a
, 'tcx
> {
194 type Target
= ConstCx
<'a
, 'tcx
>;
196 fn deref(&self) -> &Self::Target
{
203 impl<'tcx
> Validator
<'_
, 'tcx
> {
204 fn validate_candidate(&mut self, candidate
: Candidate
) -> Result
<(), Unpromotable
> {
205 let loc
= candidate
.location
;
206 let statement
= &self.body
[loc
.block
].statements
[loc
.statement_index
];
207 match &statement
.kind
{
208 StatementKind
::Assign(box (_
, Rvalue
::Ref(_
, kind
, place
))) => {
209 // We can only promote interior borrows of promotable temps (non-temps
210 // don't get promoted anyway).
211 self.validate_local(place
.local
)?
;
213 // The reference operation itself must be promotable.
214 // (Needs to come after `validate_local` to avoid ICEs.)
215 self.validate_ref(*kind
, place
)?
;
217 // We do not check all the projections (they do not get promoted anyway),
218 // but we do stay away from promoting anything involving a dereference.
219 if place
.projection
.contains(&ProjectionElem
::Deref
) {
220 return Err(Unpromotable
);
223 // We cannot promote things that need dropping, since the promoted value
224 // would not get dropped.
225 if self.qualif_local
::<qualifs
::NeedsDrop
>(place
.local
) {
226 return Err(Unpromotable
);
235 // FIXME(eddyb) maybe cache this?
236 fn qualif_local
<Q
: qualifs
::Qualif
>(&mut self, local
: Local
) -> bool
{
237 if let TempState
::Defined { location: loc, .. }
= self.temps
[local
] {
238 let num_stmts
= self.body
[loc
.block
].statements
.len();
240 if loc
.statement_index
< num_stmts
{
241 let statement
= &self.body
[loc
.block
].statements
[loc
.statement_index
];
242 match &statement
.kind
{
243 StatementKind
::Assign(box (_
, rhs
)) => qualifs
::in_rvalue
::<Q
, _
>(
245 &mut |l
| self.qualif_local
::<Q
>(l
),
250 statement
.source_info
.span
,
251 "{:?} is not an assignment",
257 let terminator
= self.body
[loc
.block
].terminator();
258 match &terminator
.kind
{
259 TerminatorKind
::Call { .. }
=> {
260 let return_ty
= self.body
.local_decls
[local
].ty
;
261 Q
::in_any_value_of_ty(&self.ccx
, return_ty
)
264 span_bug
!(terminator
.source_info
.span
, "{:?} not promotable", kind
);
269 let span
= self.body
.local_decls
[local
].source_info
.span
;
270 span_bug
!(span
, "{:?} not promotable, qualif_local shouldn't have been called", local
);
274 fn validate_local(&mut self, local
: Local
) -> Result
<(), Unpromotable
> {
275 if let TempState
::Defined { location: loc, uses, valid }
= self.temps
[local
] {
278 let block
= &self.body
[loc
.block
];
279 let num_stmts
= block
.statements
.len();
281 if loc
.statement_index
< num_stmts
{
282 let statement
= &block
.statements
[loc
.statement_index
];
283 match &statement
.kind
{
284 StatementKind
::Assign(box (_
, rhs
)) => self.validate_rvalue(rhs
),
287 statement
.source_info
.span
,
288 "{:?} is not an assignment",
294 let terminator
= block
.terminator();
295 match &terminator
.kind
{
296 TerminatorKind
::Call { func, args, .. }
=> {
297 self.validate_call(func
, args
)
299 TerminatorKind
::Yield { .. }
=> Err(Unpromotable
),
301 span_bug
!(terminator
.source_info
.span
, "{:?} not promotable", kind
);
306 self.temps
[local
] = match ok
{
307 Ok(()) => TempState
::Defined { location: loc, uses, valid: Ok(()) }
,
308 Err(_
) => TempState
::Unpromotable
,
317 fn validate_place(&mut self, place
: PlaceRef
<'tcx
>) -> Result
<(), Unpromotable
> {
318 match place
.last_projection() {
319 None
=> self.validate_local(place
.local
),
320 Some((place_base
, elem
)) => {
321 // Validate topmost projection, then recurse.
323 ProjectionElem
::Deref
=> {
324 let mut promotable
= false;
325 // We need to make sure this is a `Deref` of a local with no further projections.
326 // Discussion can be found at
327 // https://github.com/rust-lang/rust/pull/74945#discussion_r463063247
328 if let Some(local
) = place_base
.as_local() {
329 // This is a special treatment for cases like *&STATIC where STATIC is a
330 // global static variable.
331 // This pattern is generated only when global static variables are directly
332 // accessed and is qualified for promotion safely.
333 if let TempState
::Defined { location, .. }
= self.temps
[local
] {
334 let def_stmt
= self.body
[location
.block
]
336 .get(location
.statement_index
);
337 if let Some(Statement
{
339 StatementKind
::Assign(box (
341 Rvalue
::Use(Operand
::Constant(c
)),
346 if let Some(did
) = c
.check_static_ptr(self.tcx
) {
347 // Evaluating a promoted may not read statics except if it got
348 // promoted from a static (this is a CTFE check). So we
349 // can only promote static accesses inside statics.
350 if let Some(hir
::ConstContext
::Static(..)) = self.const_kind
352 if !self.tcx
.is_thread_local_static(did
) {
361 return Err(Unpromotable
);
364 ProjectionElem
::Downcast(..) => {
365 return Err(Unpromotable
);
368 ProjectionElem
::ConstantIndex { .. }
| ProjectionElem
::Subslice { .. }
=> {}
370 ProjectionElem
::Index(local
) => {
371 let mut promotable
= false;
372 // Only accept if we can predict the index and are indexing an array.
374 if let TempState
::Defined { location: loc, .. }
= self.temps
[local
] {
375 let block
= &self.body
[loc
.block
];
376 if loc
.statement_index
< block
.statements
.len() {
377 let statement
= &block
.statements
[loc
.statement_index
];
378 match &statement
.kind
{
379 StatementKind
::Assign(box (
381 Rvalue
::Use(Operand
::Constant(c
)),
382 )) => c
.literal
.try_eval_usize(self.tcx
, self.param_env
),
391 if let Some(idx
) = val
{
392 // Determine the type of the thing we are indexing.
393 let ty
= place_base
.ty(self.body
, self.tcx
).ty
;
395 ty
::Array(_
, len
) => {
396 // It's an array; determine its length.
397 if let Some(len
) = len
.try_eval_usize(self.tcx
, self.param_env
)
399 // If the index is in-bounds, go ahead.
409 return Err(Unpromotable
);
412 self.validate_local(local
)?
;
415 ProjectionElem
::Field(..) => {
416 let base_ty
= place_base
.ty(self.body
, self.tcx
).ty
;
417 if base_ty
.is_union() {
418 // No promotion of union field accesses.
419 return Err(Unpromotable
);
424 self.validate_place(place_base
)
429 fn validate_operand(&mut self, operand
: &Operand
<'tcx
>) -> Result
<(), Unpromotable
> {
431 Operand
::Copy(place
) | Operand
::Move(place
) => self.validate_place(place
.as_ref()),
433 // The qualifs for a constant (e.g. `HasMutInterior`) are checked in
434 // `validate_rvalue` upon access.
435 Operand
::Constant(c
) => {
436 if let Some(def_id
) = c
.check_static_ptr(self.tcx
) {
437 // Only allow statics (not consts) to refer to other statics.
438 // FIXME(eddyb) does this matter at all for promotion?
439 // FIXME(RalfJung) it makes little sense to not promote this in `fn`/`const fn`,
440 // and in `const` this cannot occur anyway. The only concern is that we might
441 // promote even `let x = &STATIC` which would be useless, but this applies to
442 // promotion inside statics as well.
443 let is_static
= matches
!(self.const_kind
, Some(hir
::ConstContext
::Static(_
)));
445 return Err(Unpromotable
);
448 let is_thread_local
= self.tcx
.is_thread_local_static(def_id
);
450 return Err(Unpromotable
);
459 fn validate_ref(&mut self, kind
: BorrowKind
, place
: &Place
<'tcx
>) -> Result
<(), Unpromotable
> {
461 // Reject these borrow types just to be safe.
462 // FIXME(RalfJung): could we allow them? Should we? No point in it until we have a usecase.
463 BorrowKind
::Shallow
| BorrowKind
::Unique
=> return Err(Unpromotable
),
465 BorrowKind
::Shared
=> {
466 let has_mut_interior
= self.qualif_local
::<qualifs
::HasMutInterior
>(place
.local
);
467 if has_mut_interior
{
468 return Err(Unpromotable
);
472 BorrowKind
::Mut { .. }
=> {
473 let ty
= place
.ty(self.body
, self.tcx
).ty
;
475 // In theory, any zero-sized value could be borrowed
476 // mutably without consequences. However, only &mut []
477 // is allowed right now.
478 if let ty
::Array(_
, len
) = ty
.kind() {
479 match len
.try_eval_usize(self.tcx
, self.param_env
) {
481 _
=> return Err(Unpromotable
),
484 return Err(Unpromotable
);
492 fn validate_rvalue(&mut self, rvalue
: &Rvalue
<'tcx
>) -> Result
<(), Unpromotable
> {
494 Rvalue
::Use(operand
) | Rvalue
::Repeat(operand
, _
) => {
495 self.validate_operand(operand
)?
;
498 Rvalue
::Discriminant(place
) | Rvalue
::Len(place
) => {
499 self.validate_place(place
.as_ref())?
502 Rvalue
::ThreadLocalRef(_
) => return Err(Unpromotable
),
504 // ptr-to-int casts are not possible in consts and thus not promotable
505 Rvalue
::Cast(CastKind
::PointerExposeAddress
, _
, _
) => return Err(Unpromotable
),
507 // all other casts including int-to-ptr casts are fine, they just use the integer value
509 Rvalue
::Cast(_
, operand
, _
) => {
510 self.validate_operand(operand
)?
;
513 Rvalue
::NullaryOp(op
, _
) => match op
{
515 NullOp
::AlignOf
=> {}
518 Rvalue
::ShallowInitBox(_
, _
) => return Err(Unpromotable
),
520 Rvalue
::UnaryOp(op
, operand
) => {
522 // These operations can never fail.
523 UnOp
::Neg
| UnOp
::Not
=> {}
526 self.validate_operand(operand
)?
;
529 Rvalue
::BinaryOp(op
, box (lhs
, rhs
)) | Rvalue
::CheckedBinaryOp(op
, box (lhs
, rhs
)) => {
531 let lhs_ty
= lhs
.ty(self.body
, self.tcx
);
533 if let ty
::RawPtr(_
) | ty
::FnPtr(..) = lhs_ty
.kind() {
534 // Raw and fn pointer operations are not allowed inside consts and thus not promotable.
545 return Err(Unpromotable
);
549 BinOp
::Div
| BinOp
::Rem
=> {
550 if lhs_ty
.is_integral() {
551 // Integer division: the RHS must be a non-zero const.
552 let const_val
= match rhs
{
553 Operand
::Constant(c
) => {
554 c
.literal
.try_eval_bits(self.tcx
, self.param_env
, lhs_ty
)
559 Some(x
) if x
!= 0 => {}
// okay
560 _
=> return Err(Unpromotable
), // value not known or 0 -- not okay
564 // The remaining operations can never fail.
582 self.validate_operand(lhs
)?
;
583 self.validate_operand(rhs
)?
;
586 Rvalue
::AddressOf(_
, place
) => {
587 // We accept `&raw *`, i.e., raw reborrows -- creating a raw pointer is
588 // no problem, only using it is.
589 if let Some((place_base
, ProjectionElem
::Deref
)) = place
.as_ref().last_projection()
591 let base_ty
= place_base
.ty(self.body
, self.tcx
).ty
;
592 if let ty
::Ref(..) = base_ty
.kind() {
593 return self.validate_place(place_base
);
596 return Err(Unpromotable
);
599 Rvalue
::Ref(_
, kind
, place
) => {
600 // Special-case reborrows to be more like a copy of the reference.
601 let mut place_simplified
= place
.as_ref();
602 if let Some((place_base
, ProjectionElem
::Deref
)) =
603 place_simplified
.last_projection()
605 let base_ty
= place_base
.ty(self.body
, self.tcx
).ty
;
606 if let ty
::Ref(..) = base_ty
.kind() {
607 place_simplified
= place_base
;
611 self.validate_place(place_simplified
)?
;
613 // Check that the reference is fine (using the original place!).
614 // (Needs to come after `validate_place` to avoid ICEs.)
615 self.validate_ref(*kind
, place
)?
;
618 Rvalue
::Aggregate(_
, operands
) => {
620 self.validate_operand(o
)?
;
630 callee
: &Operand
<'tcx
>,
631 args
: &[Operand
<'tcx
>],
632 ) -> Result
<(), Unpromotable
> {
633 let fn_ty
= callee
.ty(self.body
, self.tcx
);
635 // Inside const/static items, we promote all (eligible) function calls.
636 // Everywhere else, we require `#[rustc_promotable]` on the callee.
637 let promote_all_const_fn
= matches
!(
639 Some(hir
::ConstContext
::Static(_
) | hir
::ConstContext
::Const
)
641 if !promote_all_const_fn
{
642 if let ty
::FnDef(def_id
, _
) = *fn_ty
.kind() {
643 // Never promote runtime `const fn` calls of
644 // functions without `#[rustc_promotable]`.
645 if !self.tcx
.is_promotable_const_fn(def_id
) {
646 return Err(Unpromotable
);
651 let is_const_fn
= match *fn_ty
.kind() {
652 ty
::FnDef(def_id
, _
) => self.tcx
.is_const_fn_raw(def_id
),
656 return Err(Unpromotable
);
659 self.validate_operand(callee
)?
;
661 self.validate_operand(arg
)?
;
668 // FIXME(eddyb) remove the differences for promotability in `static`, `const`, `const fn`.
669 pub fn validate_candidates(
670 ccx
: &ConstCx
<'_
, '_
>,
671 temps
: &mut IndexVec
<Local
, TempState
>,
672 candidates
: &[Candidate
],
673 ) -> Vec
<Candidate
> {
674 let mut validator
= Validator { ccx, temps }
;
679 .filter(|&candidate
| validator
.validate_candidate(candidate
).is_ok())
683 struct Promoter
<'a
, 'tcx
> {
685 source
: &'a
mut Body
<'tcx
>,
686 promoted
: Body
<'tcx
>,
687 temps
: &'a
mut IndexVec
<Local
, TempState
>,
688 extra_statements
: &'a
mut Vec
<(Location
, Statement
<'tcx
>)>,
690 /// If true, all nested temps are also kept in the
691 /// source MIR, not moved to the promoted MIR.
695 impl<'a
, 'tcx
> Promoter
<'a
, 'tcx
> {
696 fn new_block(&mut self) -> BasicBlock
{
697 let span
= self.promoted
.span
;
698 self.promoted
.basic_blocks_mut().push(BasicBlockData
{
700 terminator
: Some(Terminator
{
701 source_info
: SourceInfo
::outermost(span
),
702 kind
: TerminatorKind
::Return
,
708 fn assign(&mut self, dest
: Local
, rvalue
: Rvalue
<'tcx
>, span
: Span
) {
709 let last
= self.promoted
.basic_blocks().last().unwrap();
710 let data
= &mut self.promoted
[last
];
711 data
.statements
.push(Statement
{
712 source_info
: SourceInfo
::outermost(span
),
713 kind
: StatementKind
::Assign(Box
::new((Place
::from(dest
), rvalue
))),
717 fn is_temp_kind(&self, local
: Local
) -> bool
{
718 self.source
.local_kind(local
) == LocalKind
::Temp
721 /// Copies the initialization of this temp to the
722 /// promoted MIR, recursing through temps.
723 fn promote_temp(&mut self, temp
: Local
) -> Local
{
724 let old_keep_original
= self.keep_original
;
725 let loc
= match self.temps
[temp
] {
726 TempState
::Defined { location, uses, .. }
if uses
> 0 => {
728 self.keep_original
= true;
733 span_bug
!(self.promoted
.span
, "{:?} not promotable: {:?}", temp
, state
);
736 if !self.keep_original
{
737 self.temps
[temp
] = TempState
::PromotedOut
;
740 let num_stmts
= self.source
[loc
.block
].statements
.len();
741 let new_temp
= self.promoted
.local_decls
.push(LocalDecl
::new(
742 self.source
.local_decls
[temp
].ty
,
743 self.source
.local_decls
[temp
].source_info
.span
,
746 debug
!("promote({:?} @ {:?}/{:?}, {:?})", temp
, loc
, num_stmts
, self.keep_original
);
748 // First, take the Rvalue or Call out of the source MIR,
749 // or duplicate it, depending on keep_original.
750 if loc
.statement_index
< num_stmts
{
751 let (mut rvalue
, source_info
) = {
752 let statement
= &mut self.source
[loc
.block
].statements
[loc
.statement_index
];
753 let StatementKind
::Assign(box (_
, ref mut rhs
)) = statement
.kind
else {
755 statement
.source_info
.span
,
756 "{:?} is not an assignment",
762 if self.keep_original
{
765 let unit
= Rvalue
::Use(Operand
::Constant(Box
::new(Constant
{
766 span
: statement
.source_info
.span
,
768 literal
: ConstantKind
::zero_sized(self.tcx
.types
.unit
),
770 mem
::replace(rhs
, unit
)
772 statement
.source_info
,
776 self.visit_rvalue(&mut rvalue
, loc
);
777 self.assign(new_temp
, rvalue
, source_info
.span
);
779 let terminator
= if self.keep_original
{
780 self.source
[loc
.block
].terminator().clone()
782 let terminator
= self.source
[loc
.block
].terminator_mut();
783 let target
= match terminator
.kind
{
784 TerminatorKind
::Call { target: Some(target), .. }
=> target
,
786 span_bug
!(terminator
.source_info
.span
, "{:?} not promotable", kind
);
790 source_info
: terminator
.source_info
,
791 kind
: mem
::replace(&mut terminator
.kind
, TerminatorKind
::Goto { target }
),
795 match terminator
.kind
{
796 TerminatorKind
::Call { mut func, mut args, from_hir_call, fn_span, .. }
=> {
797 self.visit_operand(&mut func
, loc
);
798 for arg
in &mut args
{
799 self.visit_operand(arg
, loc
);
802 let last
= self.promoted
.basic_blocks().last().unwrap();
803 let new_target
= self.new_block();
805 *self.promoted
[last
].terminator_mut() = Terminator
{
806 kind
: TerminatorKind
::Call
{
810 destination
: Place
::from(new_temp
),
811 target
: Some(new_target
),
815 source_info
: SourceInfo
::outermost(terminator
.source_info
.span
),
820 span_bug
!(terminator
.source_info
.span
, "{:?} not promotable", kind
);
825 self.keep_original
= old_keep_original
;
829 fn promote_candidate(mut self, candidate
: Candidate
, next_promoted_id
: usize) -> Body
<'tcx
> {
830 let def
= self.source
.source
.with_opt_param();
832 let promoted
= &mut self.promoted
;
833 let promoted_id
= Promoted
::new(next_promoted_id
);
835 let mut promoted_operand
= |ty
, span
| {
836 promoted
.span
= span
;
837 promoted
.local_decls
[RETURN_PLACE
] = LocalDecl
::new(ty
, span
);
838 let _const
= tcx
.mk_const(ty
::ConstS
{
840 kind
: ty
::ConstKind
::Unevaluated(ty
::Unevaluated
{
842 substs
: InternalSubsts
::for_item(tcx
, def
.did
, |param
, _
| {
843 if let ty
::GenericParamDefKind
::Lifetime
= param
.kind
{
844 tcx
.lifetimes
.re_erased
.into()
846 tcx
.mk_param_from_def(param
)
849 promoted
: Some(promoted_id
),
853 Operand
::Constant(Box
::new(Constant
{
856 literal
: ConstantKind
::from_const(_const
, tcx
),
859 let (blocks
, local_decls
) = self.source
.basic_blocks_and_local_decls_mut();
860 let loc
= candidate
.location
;
861 let statement
= &mut blocks
[loc
.block
].statements
[loc
.statement_index
];
862 match statement
.kind
{
863 StatementKind
::Assign(box (
865 Rvalue
::Ref(ref mut region
, borrow_kind
, ref mut place
),
867 // Use the underlying local for this (necessarily interior) borrow.
868 let ty
= local_decls
.local_decls()[place
.local
].ty
;
869 let span
= statement
.source_info
.span
;
871 let ref_ty
= tcx
.mk_ref(
872 tcx
.lifetimes
.re_erased
,
873 ty
::TypeAndMut { ty, mutbl: borrow_kind.to_mutbl_lossy() }
,
876 *region
= tcx
.lifetimes
.re_erased
;
878 let mut projection
= vec
![PlaceElem
::Deref
];
879 projection
.extend(place
.projection
);
880 place
.projection
= tcx
.intern_place_elems(&projection
);
882 // Create a temp to hold the promoted reference.
883 // This is because `*r` requires `r` to be a local,
884 // otherwise we would use the `promoted` directly.
885 let mut promoted_ref
= LocalDecl
::new(ref_ty
, span
);
886 promoted_ref
.source_info
= statement
.source_info
;
887 let promoted_ref
= local_decls
.push(promoted_ref
);
888 assert_eq
!(self.temps
.push(TempState
::Unpromotable
), promoted_ref
);
890 let promoted_ref_statement
= Statement
{
891 source_info
: statement
.source_info
,
892 kind
: StatementKind
::Assign(Box
::new((
893 Place
::from(promoted_ref
),
894 Rvalue
::Use(promoted_operand(ref_ty
, span
)),
897 self.extra_statements
.push((loc
, promoted_ref_statement
));
900 tcx
.lifetimes
.re_erased
,
903 local
: mem
::replace(&mut place
.local
, promoted_ref
),
904 projection
: List
::empty(),
912 assert_eq
!(self.new_block(), START_BLOCK
);
915 Location { block: BasicBlock::new(0), statement_index: usize::MAX }
,
918 let span
= self.promoted
.span
;
919 self.assign(RETURN_PLACE
, rvalue
, span
);
924 /// Replaces all temporaries with their promoted counterparts.
925 impl<'a
, 'tcx
> MutVisitor
<'tcx
> for Promoter
<'a
, 'tcx
> {
926 fn tcx(&self) -> TyCtxt
<'tcx
> {
930 fn visit_local(&mut self, local
: &mut Local
, _
: PlaceContext
, _
: Location
) {
931 if self.is_temp_kind(*local
) {
932 *local
= self.promote_temp(*local
);
937 pub fn promote_candidates
<'tcx
>(
938 body
: &mut Body
<'tcx
>,
940 mut temps
: IndexVec
<Local
, TempState
>,
941 candidates
: Vec
<Candidate
>,
942 ) -> IndexVec
<Promoted
, Body
<'tcx
>> {
943 // Visit candidates in reverse, in case they're nested.
944 debug
!("promote_candidates({:?})", candidates
);
946 let mut promotions
= IndexVec
::new();
948 let mut extra_statements
= vec
![];
949 for candidate
in candidates
.into_iter().rev() {
950 let Location { block, statement_index }
= candidate
.location
;
951 if let StatementKind
::Assign(box (place
, _
)) = &body
[block
].statements
[statement_index
].kind
953 if let Some(local
) = place
.as_local() {
954 if temps
[local
] == TempState
::PromotedOut
{
961 // Declare return place local so that `mir::Body::new` doesn't complain.
962 let initial_locals
= iter
::once(LocalDecl
::new(tcx
.types
.never
, body
.span
)).collect();
964 let mut scope
= body
.source_scopes
[body
.source_info(candidate
.location
).scope
].clone();
965 scope
.parent_scope
= None
;
967 let promoted
= Body
::new(
968 body
.source
, // `promoted` gets filled in below
970 IndexVec
::from_elem_n(scope
, 1),
976 body
.generator_kind(),
977 body
.tainted_by_errors
,
980 let promoter
= Promoter
{
985 extra_statements
: &mut extra_statements
,
986 keep_original
: false,
989 let mut promoted
= promoter
.promote_candidate(candidate
, promotions
.len());
990 promoted
.source
.promoted
= Some(promotions
.next_index());
991 promotions
.push(promoted
);
994 // Insert each of `extra_statements` before its indicated location, which
995 // has to be done in reverse location order, to not invalidate the rest.
996 extra_statements
.sort_by_key(|&(loc
, _
)| cmp
::Reverse(loc
));
997 for (loc
, statement
) in extra_statements
{
998 body
[loc
.block
].statements
.insert(loc
.statement_index
, statement
);
1001 // Eliminate assignments to, and drops of promoted temps.
1002 let promoted
= |index
: Local
| temps
[index
] == TempState
::PromotedOut
;
1003 for block
in body
.basic_blocks_mut() {
1004 block
.statements
.retain(|statement
| match &statement
.kind
{
1005 StatementKind
::Assign(box (place
, _
)) => {
1006 if let Some(index
) = place
.as_local() {
1012 StatementKind
::StorageLive(index
) | StatementKind
::StorageDead(index
) => {
1017 let terminator
= block
.terminator_mut();
1018 if let TerminatorKind
::Drop { place, target, .. }
= &terminator
.kind
{
1019 if let Some(index
) = place
.as_local() {
1020 if promoted(index
) {
1021 terminator
.kind
= TerminatorKind
::Goto { target: *target }
;
1030 /// This function returns `true` if the function being called in the array
1031 /// repeat expression is a `const` function.
1032 pub fn is_const_fn_in_array_repeat_expression
<'tcx
>(
1033 ccx
: &ConstCx
<'_
, 'tcx
>,
1034 place
: &Place
<'tcx
>,
1037 match place
.as_local() {
1038 // rule out cases such as: `let my_var = some_fn(); [my_var; N]`
1039 Some(local
) if body
.local_decls
[local
].is_user_variable() => return false,
1040 None
=> return false,
1044 for block
in body
.basic_blocks() {
1045 if let Some(Terminator { kind: TerminatorKind::Call { func, destination, .. }
, .. }) =
1048 if let Operand
::Constant(box Constant { literal, .. }
) = func
{
1049 if let ty
::FnDef(def_id
, _
) = *literal
.ty().kind() {
1050 if destination
== place
{
1051 if ccx
.tcx
.is_const_fn(def_id
) {