[rustc.git] / compiler / rustc_mir_transform / src / lower_intrinsics.rs

//! Lowers intrinsic calls

use crate::MirPass;
use rustc_middle::mir::*;
use rustc_middle::ty::subst::SubstsRef;
use rustc_middle::ty::{self, Ty, TyCtxt};
use rustc_span::symbol::{sym, Symbol};
use rustc_span::Span;
use rustc_target::abi::{FieldIdx, VariantIdx};

pub struct LowerIntrinsics;

impl<'tcx> MirPass<'tcx> for LowerIntrinsics {
    fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
        let local_decls = &body.local_decls;
        for block in body.basic_blocks.as_mut() {
            let terminator = block.terminator.as_mut().unwrap();
            if let TerminatorKind::Call { func, args, destination, target, .. } =
                &mut terminator.kind
            {
                let func_ty = func.ty(local_decls, tcx);
                let Some((intrinsic_name, substs)) = resolve_rust_intrinsic(tcx, func_ty) else {
                    continue;
                };
                match intrinsic_name {
                    sym::unreachable => {
                        terminator.kind = TerminatorKind::Unreachable;
                    }
                    sym::forget => {
                        if let Some(target) = *target {
                            block.statements.push(Statement {
                                source_info: terminator.source_info,
                                kind: StatementKind::Assign(Box::new((
                                    *destination,
                                    Rvalue::Use(Operand::Constant(Box::new(Constant {
                                        span: terminator.source_info.span,
                                        user_ty: None,
                                        literal: ConstantKind::zero_sized(tcx.types.unit),
                                    }))),
                                ))),
                            });
                            terminator.kind = TerminatorKind::Goto { target };
                        }
                    }
                    sym::copy_nonoverlapping => {
                        let target = target.unwrap();
                        let mut args = args.drain(..);
                        block.statements.push(Statement {
                            source_info: terminator.source_info,
                            kind: StatementKind::Intrinsic(Box::new(
                                NonDivergingIntrinsic::CopyNonOverlapping(
                                    rustc_middle::mir::CopyNonOverlapping {
                                        src: args.next().unwrap(),
                                        dst: args.next().unwrap(),
                                        count: args.next().unwrap(),
                                    },
                                ),
                            )),
                        });
                        assert_eq!(
                            args.next(),
                            None,
                            "Extra argument for copy_non_overlapping intrinsic"
                        );
                        drop(args);
                        terminator.kind = TerminatorKind::Goto { target };
                    }
                    sym::assume => {
                        let target = target.unwrap();
                        let mut args = args.drain(..);
                        block.statements.push(Statement {
                            source_info: terminator.source_info,
                            kind: StatementKind::Intrinsic(Box::new(
                                NonDivergingIntrinsic::Assume(args.next().unwrap()),
                            )),
                        });
                        assert_eq!(
                            args.next(),
                            None,
                            "Extra argument for copy_non_overlapping intrinsic"
                        );
                        drop(args);
                        terminator.kind = TerminatorKind::Goto { target };
                    }
                    sym::wrapping_add | sym::wrapping_sub | sym::wrapping_mul => {
                        if let Some(target) = *target {
                            let lhs;
                            let rhs;
                            {
                                let mut args = args.drain(..);
                                lhs = args.next().unwrap();
                                rhs = args.next().unwrap();
                            }
                            let bin_op = match intrinsic_name {
                                sym::wrapping_add => BinOp::Add,
                                sym::wrapping_sub => BinOp::Sub,
                                sym::wrapping_mul => BinOp::Mul,
                                _ => bug!("unexpected intrinsic"),
                            };
                            block.statements.push(Statement {
                                source_info: terminator.source_info,
                                kind: StatementKind::Assign(Box::new((
                                    *destination,
                                    Rvalue::BinaryOp(bin_op, Box::new((lhs, rhs))),
                                ))),
                            });
                            terminator.kind = TerminatorKind::Goto { target };
                        }
                    }
                    sym::add_with_overflow | sym::sub_with_overflow | sym::mul_with_overflow => {
                        if let Some(target) = *target {
                            let lhs;
                            let rhs;
                            {
                                let mut args = args.drain(..);
                                lhs = args.next().unwrap();
                                rhs = args.next().unwrap();
                            }
                            let bin_op = match intrinsic_name {
                                sym::add_with_overflow => BinOp::Add,
                                sym::sub_with_overflow => BinOp::Sub,
                                sym::mul_with_overflow => BinOp::Mul,
                                _ => bug!("unexpected intrinsic"),
                            };
                            block.statements.push(Statement {
                                source_info: terminator.source_info,
                                kind: StatementKind::Assign(Box::new((
                                    *destination,
                                    Rvalue::CheckedBinaryOp(bin_op, Box::new((lhs, rhs))),
                                ))),
                            });
                            terminator.kind = TerminatorKind::Goto { target };
                        }
                    }
                    sym::size_of | sym::min_align_of => {
                        if let Some(target) = *target {
                            let tp_ty = substs.type_at(0);
                            let null_op = match intrinsic_name {
                                sym::size_of => NullOp::SizeOf,
                                sym::min_align_of => NullOp::AlignOf,
                                _ => bug!("unexpected intrinsic"),
                            };
                            block.statements.push(Statement {
                                source_info: terminator.source_info,
                                kind: StatementKind::Assign(Box::new((
                                    *destination,
                                    Rvalue::NullaryOp(null_op, tp_ty),
                                ))),
                            });
                            terminator.kind = TerminatorKind::Goto { target };
                        }
                    }
                    sym::read_via_copy => {
                        let [arg] = args.as_slice() else {
                            span_bug!(terminator.source_info.span, "Wrong number of arguments");
                        };
                        let derefed_place =
                            if let Some(place) = arg.place() && let Some(local) = place.as_local() {
                                tcx.mk_place_deref(local.into())
                            } else {
                                span_bug!(terminator.source_info.span, "Only passing a local is supported");
                            };
                        terminator.kind = match *target {
                            None => {
                                // No target means this read something uninhabited,
                                // so it must be unreachable, and we don't need to
                                // preserve the assignment either.
                                TerminatorKind::Unreachable
                            }
                            Some(target) => {
                                block.statements.push(Statement {
                                    source_info: terminator.source_info,
                                    kind: StatementKind::Assign(Box::new((
                                        *destination,
                                        Rvalue::Use(Operand::Copy(derefed_place)),
                                    ))),
                                });
                                TerminatorKind::Goto { target }
                            }
                        }
                    }
                    sym::discriminant_value => {
                        if let (Some(target), Some(arg)) = (*target, args[0].place()) {
                            let arg = tcx.mk_place_deref(arg);
                            block.statements.push(Statement {
                                source_info: terminator.source_info,
                                kind: StatementKind::Assign(Box::new((
                                    *destination,
                                    Rvalue::Discriminant(arg),
                                ))),
                            });
                            terminator.kind = TerminatorKind::Goto { target };
                        }
                    }
                    sym::option_payload_ptr => {
                        if let (Some(target), Some(arg)) = (*target, args[0].place()) {
                            let ty::RawPtr(ty::TypeAndMut { ty: dest_ty, .. }) =
                                destination.ty(local_decls, tcx).ty.kind()
                            else { bug!(); };

                            block.statements.push(Statement {
                                source_info: terminator.source_info,
                                kind: StatementKind::Assign(Box::new((
                                    *destination,
                                    Rvalue::AddressOf(
                                        Mutability::Not,
                                        arg.project_deeper(
                                            &[
                                                PlaceElem::Deref,
                                                PlaceElem::Downcast(
                                                    Some(sym::Some),
                                                    VariantIdx::from_u32(1),
                                                ),
                                                PlaceElem::Field(FieldIdx::from_u32(0), *dest_ty),
                                            ],
                                            tcx,
                                        ),
                                    ),
                                ))),
                            });
                            terminator.kind = TerminatorKind::Goto { target };
                        }
                    }
                    sym::transmute => {
                        let dst_ty = destination.ty(local_decls, tcx).ty;
                        let Ok([arg]) = <[_; 1]>::try_from(std::mem::take(args)) else {
                            span_bug!(
                                terminator.source_info.span,
                                "Wrong number of arguments for transmute intrinsic",
                            );
                        };

                        // Always emit the cast, even if we transmute to an uninhabited type,
                        // because that lets CTFE and codegen generate better error messages
                        // when such a transmute actually ends up reachable.
                        block.statements.push(Statement {
                            source_info: terminator.source_info,
                            kind: StatementKind::Assign(Box::new((
                                *destination,
                                Rvalue::Cast(CastKind::Transmute, arg, dst_ty),
                            ))),
                        });

                        if let Some(target) = *target {
                            terminator.kind = TerminatorKind::Goto { target };
                        } else {
                            terminator.kind = TerminatorKind::Unreachable;
                        }
                    }
                    _ if intrinsic_name.as_str().starts_with("simd_shuffle") => {
                        validate_simd_shuffle(tcx, args, terminator.source_info.span);
                    }
                    _ => {}
                }
            }
        }
    }
}

fn resolve_rust_intrinsic<'tcx>(
    tcx: TyCtxt<'tcx>,
    func_ty: Ty<'tcx>,
) -> Option<(Symbol, SubstsRef<'tcx>)> {
    if let ty::FnDef(def_id, substs) = *func_ty.kind() {
        if tcx.is_intrinsic(def_id) {
            return Some((tcx.item_name(def_id), substs));
        }
    }
    None
}

fn validate_simd_shuffle<'tcx>(tcx: TyCtxt<'tcx>, args: &[Operand<'tcx>], span: Span) {
    match &args[2] {
        Operand::Constant(_) => {} // all good
        _ => {
            let msg = "last argument of `simd_shuffle` is required to be a `const` item";
            tcx.sess.span_err(span, msg);
        }
    }
}
Commit	Line	Data
fc512014 XL	1	//! Lowers intrinsic calls
fc512014 XL	2
c295e0f8	3	use crate::MirPass;
fc512014 XL	4	use rustc_middle::mir::*;
	5	use rustc_middle::ty::subst::SubstsRef;
	6	use rustc_middle::ty::{self, Ty, TyCtxt};
	7	use rustc_span::symbol::{sym, Symbol};
17df50a5	8	use rustc_span::Span;
353b0b11	9	use rustc_target::abi::{FieldIdx, VariantIdx};
fc512014 XL	10
	11	pub struct LowerIntrinsics;
	12
	13	impl<'tcx> MirPass<'tcx> for LowerIntrinsics {
	14	fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
064997fb FG	15	let local_decls = &body.local_decls;
064997fb FG	16	for block in body.basic_blocks.as_mut() {
fc512014	17	let terminator = block.terminator.as_mut().unwrap();
923072b8 FG	18	if let TerminatorKind::Call { func, args, destination, target, .. } =
	19	&mut terminator.kind
	20	{
fc512014	21	let func_ty = func.ty(local_decls, tcx);
5e7ed085 FG	22	let Some((intrinsic_name, substs)) = resolve_rust_intrinsic(tcx, func_ty) else {
5e7ed085 FG	23	continue;
fc512014 XL	24	};
	25	match intrinsic_name {
	26	sym::unreachable => {
	27	terminator.kind = TerminatorKind::Unreachable;
	28	}
	29	sym::forget => {
923072b8	30	if let Some(target) = *target {
fc512014 XL	31	block.statements.push(Statement {
fc512014 XL	32	source_info: terminator.source_info,
94222f64	33	kind: StatementKind::Assign(Box::new((
923072b8	34	*destination,
94222f64	35	Rvalue::Use(Operand::Constant(Box::new(Constant {
fc512014 XL	36	span: terminator.source_info.span,
fc512014 XL	37	user_ty: None,
923072b8	38	literal: ConstantKind::zero_sized(tcx.types.unit),
94222f64 XL	39	}))),
94222f64 XL	40	))),
fc512014 XL	41	});
	42	terminator.kind = TerminatorKind::Goto { target };
	43	}
	44	}
6a06907d	45	sym::copy_nonoverlapping => {
923072b8	46	let target = target.unwrap();
6a06907d XL	47	let mut args = args.drain(..);
	48	block.statements.push(Statement {
	49	source_info: terminator.source_info,
f2b60f7d FG	50	kind: StatementKind::Intrinsic(Box::new(
	51	NonDivergingIntrinsic::CopyNonOverlapping(
	52	rustc_middle::mir::CopyNonOverlapping {
	53	src: args.next().unwrap(),
	54	dst: args.next().unwrap(),
	55	count: args.next().unwrap(),
	56	},
	57	),
	58	)),
	59	});
	60	assert_eq!(
	61	args.next(),
	62	None,
	63	"Extra argument for copy_non_overlapping intrinsic"
	64	);
	65	drop(args);
	66	terminator.kind = TerminatorKind::Goto { target };
	67	}
	68	sym::assume => {
	69	let target = target.unwrap();
	70	let mut args = args.drain(..);
	71	block.statements.push(Statement {
	72	source_info: terminator.source_info,
	73	kind: StatementKind::Intrinsic(Box::new(
	74	NonDivergingIntrinsic::Assume(args.next().unwrap()),
94222f64	75	)),
6a06907d XL	76	});
	77	assert_eq!(
	78	args.next(),
	79	None,
	80	"Extra argument for copy_non_overlapping intrinsic"
	81	);
	82	drop(args);
	83	terminator.kind = TerminatorKind::Goto { target };
	84	}
fc512014	85	sym::wrapping_add \| sym::wrapping_sub \| sym::wrapping_mul => {
923072b8	86	if let Some(target) = *target {
fc512014 XL	87	let lhs;
	88	let rhs;
	89	{
	90	let mut args = args.drain(..);
	91	lhs = args.next().unwrap();
	92	rhs = args.next().unwrap();
	93	}
	94	let bin_op = match intrinsic_name {
	95	sym::wrapping_add => BinOp::Add,
	96	sym::wrapping_sub => BinOp::Sub,
	97	sym::wrapping_mul => BinOp::Mul,
	98	_ => bug!("unexpected intrinsic"),
	99	};
	100	block.statements.push(Statement {
	101	source_info: terminator.source_info,
94222f64	102	kind: StatementKind::Assign(Box::new((
923072b8	103	*destination,
94222f64 XL	104	Rvalue::BinaryOp(bin_op, Box::new((lhs, rhs))),
94222f64 XL	105	))),
fc512014 XL	106	});
	107	terminator.kind = TerminatorKind::Goto { target };
	108	}
	109	}
	110	sym::add_with_overflow \| sym::sub_with_overflow \| sym::mul_with_overflow => {
9ffffee4 FG	111	if let Some(target) = *target {
	112	let lhs;
	113	let rhs;
	114	{
	115	let mut args = args.drain(..);
	116	lhs = args.next().unwrap();
	117	rhs = args.next().unwrap();
	118	}
	119	let bin_op = match intrinsic_name {
	120	sym::add_with_overflow => BinOp::Add,
	121	sym::sub_with_overflow => BinOp::Sub,
	122	sym::mul_with_overflow => BinOp::Mul,
	123	_ => bug!("unexpected intrinsic"),
	124	};
	125	block.statements.push(Statement {
	126	source_info: terminator.source_info,
	127	kind: StatementKind::Assign(Box::new((
	128	*destination,
	129	Rvalue::CheckedBinaryOp(bin_op, Box::new((lhs, rhs))),
	130	))),
	131	});
	132	terminator.kind = TerminatorKind::Goto { target };
	133	}
fc512014	134	}
c295e0f8	135	sym::size_of \| sym::min_align_of => {
923072b8	136	if let Some(target) = *target {
fc512014	137	let tp_ty = substs.type_at(0);
c295e0f8 XL	138	let null_op = match intrinsic_name {
	139	sym::size_of => NullOp::SizeOf,
	140	sym::min_align_of => NullOp::AlignOf,
	141	_ => bug!("unexpected intrinsic"),
	142	};
fc512014 XL	143	block.statements.push(Statement {
fc512014 XL	144	source_info: terminator.source_info,
94222f64	145	kind: StatementKind::Assign(Box::new((
923072b8	146	*destination,
c295e0f8	147	Rvalue::NullaryOp(null_op, tp_ty),
94222f64	148	))),
fc512014 XL	149	});
	150	terminator.kind = TerminatorKind::Goto { target };
	151	}
	152	}
353b0b11 FG	153	sym::read_via_copy => {
	154	let [arg] = args.as_slice() else {
	155	span_bug!(terminator.source_info.span, "Wrong number of arguments");
	156	};
	157	let derefed_place =
	158	if let Some(place) = arg.place() && let Some(local) = place.as_local() {
	159	tcx.mk_place_deref(local.into())
	160	} else {
	161	span_bug!(terminator.source_info.span, "Only passing a local is supported");
	162	};
	163	terminator.kind = match *target {
	164	None => {
	165	// No target means this read something uninhabited,
	166	// so it must be unreachable, and we don't need to
	167	// preserve the assignment either.
	168	TerminatorKind::Unreachable
	169	}
	170	Some(target) => {
	171	block.statements.push(Statement {
	172	source_info: terminator.source_info,
	173	kind: StatementKind::Assign(Box::new((
	174	*destination,
	175	Rvalue::Use(Operand::Copy(derefed_place)),
	176	))),
	177	});
	178	TerminatorKind::Goto { target }
	179	}
	180	}
	181	}
fc512014	182	sym::discriminant_value => {
923072b8	183	if let (Some(target), Some(arg)) = (*target, args[0].place()) {
fc512014 XL	184	let arg = tcx.mk_place_deref(arg);
	185	block.statements.push(Statement {
	186	source_info: terminator.source_info,
94222f64	187	kind: StatementKind::Assign(Box::new((
923072b8	188	*destination,
fc512014	189	Rvalue::Discriminant(arg),
94222f64	190	))),
fc512014 XL	191	});
	192	terminator.kind = TerminatorKind::Goto { target };
	193	}
	194	}
353b0b11 FG	195	sym::option_payload_ptr => {
	196	if let (Some(target), Some(arg)) = (*target, args[0].place()) {
	197	let ty::RawPtr(ty::TypeAndMut { ty: dest_ty, .. }) =
	198	destination.ty(local_decls, tcx).ty.kind()
	199	else { bug!(); };
	200
	201	block.statements.push(Statement {
	202	source_info: terminator.source_info,
	203	kind: StatementKind::Assign(Box::new((
	204	*destination,
	205	Rvalue::AddressOf(
	206	Mutability::Not,
	207	arg.project_deeper(
	208	&[
	209	PlaceElem::Deref,
	210	PlaceElem::Downcast(
	211	Some(sym::Some),
	212	VariantIdx::from_u32(1),
	213	),
	214	PlaceElem::Field(FieldIdx::from_u32(0), *dest_ty),
	215	],
	216	tcx,
	217	),
	218	),
	219	))),
	220	});
	221	terminator.kind = TerminatorKind::Goto { target };
	222	}
	223	}
	224	sym::transmute => {
	225	let dst_ty = destination.ty(local_decls, tcx).ty;
	226	let Ok([arg]) = <[_; 1]>::try_from(std::mem::take(args)) else {
	227	span_bug!(
	228	terminator.source_info.span,
	229	"Wrong number of arguments for transmute intrinsic",
	230	);
	231	};
	232
	233	// Always emit the cast, even if we transmute to an uninhabited type,
	234	// because that lets CTFE and codegen generate better error messages
	235	// when such a transmute actually ends up reachable.
	236	block.statements.push(Statement {
	237	source_info: terminator.source_info,
	238	kind: StatementKind::Assign(Box::new((
	239	*destination,
	240	Rvalue::Cast(CastKind::Transmute, arg, dst_ty),
	241	))),
	242	});
	243
	244	if let Some(target) = *target {
	245	terminator.kind = TerminatorKind::Goto { target };
	246	} else {
	247	terminator.kind = TerminatorKind::Unreachable;
	248	}
	249	}
17df50a5 XL	250	_ if intrinsic_name.as_str().starts_with("simd_shuffle") => {
	251	validate_simd_shuffle(tcx, args, terminator.source_info.span);
	252	}
fc512014 XL	253	_ => {}
	254	}
	255	}
	256	}
	257	}
	258	}
	259
a2a8927a	260	fn resolve_rust_intrinsic<'tcx>(
fc512014 XL	261	tcx: TyCtxt<'tcx>,
	262	func_ty: Ty<'tcx>,
	263	) -> Option<(Symbol, SubstsRef<'tcx>)> {
	264	if let ty::FnDef(def_id, substs) = *func_ty.kind() {
923072b8	265	if tcx.is_intrinsic(def_id) {
fc512014 XL	266	return Some((tcx.item_name(def_id), substs));
	267	}
	268	}
	269	None
	270	}
17df50a5	271
a2a8927a	272	fn validate_simd_shuffle<'tcx>(tcx: TyCtxt<'tcx>, args: &[Operand<'tcx>], span: Span) {
17df50a5 XL	273	match &args[2] {
	274	Operand::Constant(_) => {} // all good
	275	_ => {
94222f64 XL	276	let msg = "last argument of `simd_shuffle` is required to be a `const` item";
94222f64 XL	277	tcx.sess.span_err(span, msg);
17df50a5 XL	278	}
	279	}
	280	}