1 use cranelift_codegen
::isa
::TargetFrontendConfig
;
2 use rustc_index
::vec
::IndexVec
;
3 use rustc_middle
::ty
::layout
::{
4 FnAbiError
, FnAbiOfHelpers
, FnAbiRequest
, LayoutError
, LayoutOfHelpers
,
6 use rustc_middle
::ty
::SymbolName
;
7 use rustc_target
::abi
::call
::FnAbi
;
8 use rustc_target
::abi
::{Integer, Primitive}
;
9 use rustc_target
::spec
::{HasTargetSpec, Target}
;
11 use crate::constant
::ConstantCx
;
12 use crate::prelude
::*;
14 pub(crate) fn pointer_ty(tcx
: TyCtxt
<'_
>) -> types
::Type
{
15 match tcx
.data_layout
.pointer_size
.bits() {
19 bits
=> bug
!("ptr_sized_integer: unknown pointer bit size {}", bits
),
23 pub(crate) fn scalar_to_clif_type(tcx
: TyCtxt
<'_
>, scalar
: Scalar
) -> Type
{
25 Primitive
::Int(int
, _sign
) => match int
{
26 Integer
::I8
=> types
::I8
,
27 Integer
::I16
=> types
::I16
,
28 Integer
::I32
=> types
::I32
,
29 Integer
::I64
=> types
::I64
,
30 Integer
::I128
=> types
::I128
,
32 Primitive
::F32
=> types
::F32
,
33 Primitive
::F64
=> types
::F64
,
34 Primitive
::Pointer
=> pointer_ty(tcx
),
38 fn clif_type_from_ty
<'tcx
>(tcx
: TyCtxt
<'tcx
>, ty
: Ty
<'tcx
>) -> Option
<types
::Type
> {
39 Some(match ty
.kind() {
40 ty
::Bool
=> types
::I8
,
41 ty
::Uint(size
) => match size
{
42 UintTy
::U8
=> types
::I8
,
43 UintTy
::U16
=> types
::I16
,
44 UintTy
::U32
=> types
::I32
,
45 UintTy
::U64
=> types
::I64
,
46 UintTy
::U128
=> types
::I128
,
47 UintTy
::Usize
=> pointer_ty(tcx
),
49 ty
::Int(size
) => match size
{
50 IntTy
::I8
=> types
::I8
,
51 IntTy
::I16
=> types
::I16
,
52 IntTy
::I32
=> types
::I32
,
53 IntTy
::I64
=> types
::I64
,
54 IntTy
::I128
=> types
::I128
,
55 IntTy
::Isize
=> pointer_ty(tcx
),
57 ty
::Char
=> types
::I32
,
58 ty
::Float(size
) => match size
{
59 FloatTy
::F32
=> types
::F32
,
60 FloatTy
::F64
=> types
::F64
,
62 ty
::FnPtr(_
) => pointer_ty(tcx
),
63 ty
::RawPtr(TypeAndMut { ty: pointee_ty, mutbl: _ }
) | ty
::Ref(_
, pointee_ty
, _
) => {
64 if has_ptr_meta(tcx
, *pointee_ty
) {
70 ty
::Adt(adt_def
, _
) if adt_def
.repr
.simd() => {
71 let (element
, count
) = match &tcx
.layout_of(ParamEnv
::reveal_all().and(ty
)).unwrap().abi
73 Abi
::Vector { element, count }
=> (element
.clone(), *count
),
77 match scalar_to_clif_type(tcx
, element
).by(u16::try_from(count
).unwrap()) {
78 // Cranelift currently only implements icmp for 128bit vectors.
79 Some(vector_ty
) if vector_ty
.bits() == 128 => vector_ty
,
83 ty
::Param(_
) => bug
!("ty param {:?}", ty
),
88 fn clif_pair_type_from_ty
<'tcx
>(
91 ) -> Option
<(types
::Type
, types
::Type
)> {
92 Some(match ty
.kind() {
93 ty
::Tuple(substs
) if substs
.len() == 2 => {
94 let mut types
= substs
.types();
95 let a
= clif_type_from_ty(tcx
, types
.next().unwrap())?
;
96 let b
= clif_type_from_ty(tcx
, types
.next().unwrap())?
;
97 if a
.is_vector() || b
.is_vector() {
102 ty
::RawPtr(TypeAndMut { ty: pointee_ty, mutbl: _ }
) | ty
::Ref(_
, pointee_ty
, _
) => {
103 if has_ptr_meta(tcx
, *pointee_ty
) {
104 (pointer_ty(tcx
), pointer_ty(tcx
))
113 /// Is a pointer to this type a fat ptr?
114 pub(crate) fn has_ptr_meta
<'tcx
>(tcx
: TyCtxt
<'tcx
>, ty
: Ty
<'tcx
>) -> bool
{
115 let ptr_ty
= tcx
.mk_ptr(TypeAndMut { ty, mutbl: rustc_hir::Mutability::Not }
);
116 match &tcx
.layout_of(ParamEnv
::reveal_all().and(ptr_ty
)).unwrap().abi
{
117 Abi
::Scalar(_
) => false,
118 Abi
::ScalarPair(_
, _
) => true,
119 abi
=> unreachable
!("Abi of ptr to {:?} is {:?}???", ty
, abi
),
123 pub(crate) fn codegen_icmp_imm(
124 fx
: &mut FunctionCx
<'_
, '_
, '_
>,
129 let lhs_ty
= fx
.bcx
.func
.dfg
.value_type(lhs
);
130 if lhs_ty
== types
::I128
{
131 // FIXME legalize `icmp_imm.i128` in Cranelift
133 let (lhs_lsb
, lhs_msb
) = fx
.bcx
.ins().isplit(lhs
);
134 let (rhs_lsb
, rhs_msb
) = (rhs
as u128
as u64 as i64, (rhs
as u128
>> 64) as u64 as i64);
138 let lsb_eq
= fx
.bcx
.ins().icmp_imm(IntCC
::Equal
, lhs_lsb
, rhs_lsb
);
139 let msb_eq
= fx
.bcx
.ins().icmp_imm(IntCC
::Equal
, lhs_msb
, rhs_msb
);
140 fx
.bcx
.ins().band(lsb_eq
, msb_eq
)
143 let lsb_ne
= fx
.bcx
.ins().icmp_imm(IntCC
::NotEqual
, lhs_lsb
, rhs_lsb
);
144 let msb_ne
= fx
.bcx
.ins().icmp_imm(IntCC
::NotEqual
, lhs_msb
, rhs_msb
);
145 fx
.bcx
.ins().bor(lsb_ne
, msb_ne
)
154 let msb_eq
= fx
.bcx
.ins().icmp_imm(IntCC
::Equal
, lhs_msb
, rhs_msb
);
155 let lsb_cc
= fx
.bcx
.ins().icmp_imm(intcc
, lhs_lsb
, rhs_lsb
);
156 let msb_cc
= fx
.bcx
.ins().icmp_imm(intcc
, lhs_msb
, rhs_msb
);
158 fx
.bcx
.ins().select(msb_eq
, lsb_cc
, msb_cc
)
162 let rhs
= i64::try_from(rhs
).expect("codegen_icmp_imm rhs out of range for <128bit int");
163 fx
.bcx
.ins().icmp_imm(intcc
, lhs
, rhs
)
167 pub(crate) fn type_min_max_value(
168 bcx
: &mut FunctionBuilder
<'_
>,
171 ) -> (Value
, Value
) {
172 assert
!(ty
.is_int());
174 if ty
== types
::I128
{
176 let min
= i128
::MIN
as u128
;
177 let min_lsb
= bcx
.ins().iconst(types
::I64
, min
as u64 as i64);
178 let min_msb
= bcx
.ins().iconst(types
::I64
, (min
>> 64) as u64 as i64);
179 let min
= bcx
.ins().iconcat(min_lsb
, min_msb
);
181 let max
= i128
::MAX
as u128
;
182 let max_lsb
= bcx
.ins().iconst(types
::I64
, max
as u64 as i64);
183 let max_msb
= bcx
.ins().iconst(types
::I64
, (max
>> 64) as u64 as i64);
184 let max
= bcx
.ins().iconcat(max_lsb
, max_msb
);
188 let min_half
= bcx
.ins().iconst(types
::I64
, 0);
189 let min
= bcx
.ins().iconcat(min_half
, min_half
);
191 let max_half
= bcx
.ins().iconst(types
::I64
, u64::MAX
as i64);
192 let max
= bcx
.ins().iconcat(max_half
, max_half
);
198 let min
= match (ty
, signed
) {
199 (types
::I8
, false) | (types
::I16
, false) | (types
::I32
, false) | (types
::I64
, false) => {
202 (types
::I8
, true) => i64::from(i8::MIN
),
203 (types
::I16
, true) => i64::from(i16::MIN
),
204 (types
::I32
, true) => i64::from(i32::MIN
),
205 (types
::I64
, true) => i64::MIN
,
209 let max
= match (ty
, signed
) {
210 (types
::I8
, false) => i64::from(u8::MAX
),
211 (types
::I16
, false) => i64::from(u16::MAX
),
212 (types
::I32
, false) => i64::from(u32::MAX
),
213 (types
::I64
, false) => u64::MAX
as i64,
214 (types
::I8
, true) => i64::from(i8::MAX
),
215 (types
::I16
, true) => i64::from(i16::MAX
),
216 (types
::I32
, true) => i64::from(i32::MAX
),
217 (types
::I64
, true) => i64::MAX
,
221 let (min
, max
) = (bcx
.ins().iconst(ty
, min
), bcx
.ins().iconst(ty
, max
));
226 pub(crate) fn type_sign(ty
: Ty
<'_
>) -> bool
{
228 ty
::Ref(..) | ty
::RawPtr(..) | ty
::FnPtr(..) | ty
::Char
| ty
::Uint(..) | ty
::Bool
=> false,
230 ty
::Float(..) => false, // `signed` is unused for floats
231 _
=> panic
!("{}", ty
),
235 pub(crate) struct FunctionCx
<'m
, 'clif
, 'tcx
: 'm
> {
236 pub(crate) cx
: &'clif
mut crate::CodegenCx
<'tcx
>,
237 pub(crate) module
: &'m
mut dyn Module
,
238 pub(crate) tcx
: TyCtxt
<'tcx
>,
239 pub(crate) target_config
: TargetFrontendConfig
, // Cached from module
240 pub(crate) pointer_type
: Type
, // Cached from module
241 pub(crate) constants_cx
: ConstantCx
,
243 pub(crate) instance
: Instance
<'tcx
>,
244 pub(crate) symbol_name
: SymbolName
<'tcx
>,
245 pub(crate) mir
: &'tcx Body
<'tcx
>,
246 pub(crate) fn_abi
: Option
<&'tcx FnAbi
<'tcx
, Ty
<'tcx
>>>,
248 pub(crate) bcx
: FunctionBuilder
<'clif
>,
249 pub(crate) block_map
: IndexVec
<BasicBlock
, Block
>,
250 pub(crate) local_map
: IndexVec
<Local
, CPlace
<'tcx
>>,
252 /// When `#[track_caller]` is used, the implicit caller location is stored in this variable.
253 pub(crate) caller_location
: Option
<CValue
<'tcx
>>,
255 pub(crate) clif_comments
: crate::pretty_clif
::CommentWriter
,
256 pub(crate) source_info_set
: indexmap
::IndexSet
<SourceInfo
>,
258 /// This should only be accessed by `CPlace::new_var`.
259 pub(crate) next_ssa_var
: u32,
262 impl<'tcx
> LayoutOfHelpers
<'tcx
> for FunctionCx
<'_
, '_
, 'tcx
> {
263 type LayoutOfResult
= TyAndLayout
<'tcx
>;
266 fn handle_layout_err(&self, err
: LayoutError
<'tcx
>, span
: Span
, ty
: Ty
<'tcx
>) -> ! {
267 RevealAllLayoutCx(self.tcx
).handle_layout_err(err
, span
, ty
)
271 impl<'tcx
> FnAbiOfHelpers
<'tcx
> for FunctionCx
<'_
, '_
, 'tcx
> {
272 type FnAbiOfResult
= &'tcx FnAbi
<'tcx
, Ty
<'tcx
>>;
275 fn handle_fn_abi_err(
277 err
: FnAbiError
<'tcx
>,
279 fn_abi_request
: FnAbiRequest
<'tcx
>,
281 RevealAllLayoutCx(self.tcx
).handle_fn_abi_err(err
, span
, fn_abi_request
)
285 impl<'tcx
> layout
::HasTyCtxt
<'tcx
> for FunctionCx
<'_
, '_
, 'tcx
> {
286 fn tcx
<'b
>(&'b
self) -> TyCtxt
<'tcx
> {
291 impl<'tcx
> rustc_target
::abi
::HasDataLayout
for FunctionCx
<'_
, '_
, 'tcx
> {
292 fn data_layout(&self) -> &rustc_target
::abi
::TargetDataLayout
{
293 &self.tcx
.data_layout
297 impl<'tcx
> layout
::HasParamEnv
<'tcx
> for FunctionCx
<'_
, '_
, 'tcx
> {
298 fn param_env(&self) -> ParamEnv
<'tcx
> {
299 ParamEnv
::reveal_all()
303 impl<'tcx
> HasTargetSpec
for FunctionCx
<'_
, '_
, 'tcx
> {
304 fn target_spec(&self) -> &Target
{
305 &self.tcx
.sess
.target
309 impl<'tcx
> FunctionCx
<'_
, '_
, 'tcx
> {
310 pub(crate) fn monomorphize
<T
>(&self, value
: T
) -> T
312 T
: TypeFoldable
<'tcx
> + Copy
,
314 self.instance
.subst_mir_and_normalize_erasing_regions(
316 ty
::ParamEnv
::reveal_all(),
321 pub(crate) fn clif_type(&self, ty
: Ty
<'tcx
>) -> Option
<Type
> {
322 clif_type_from_ty(self.tcx
, ty
)
325 pub(crate) fn clif_pair_type(&self, ty
: Ty
<'tcx
>) -> Option
<(Type
, Type
)> {
326 clif_pair_type_from_ty(self.tcx
, ty
)
329 pub(crate) fn get_block(&self, bb
: BasicBlock
) -> Block
{
330 *self.block_map
.get(bb
).unwrap()
333 pub(crate) fn get_local_place(&mut self, local
: Local
) -> CPlace
<'tcx
> {
334 *self.local_map
.get(local
).unwrap_or_else(|| {
335 panic
!("Local {:?} doesn't exist", local
);
339 pub(crate) fn set_debug_loc(&mut self, source_info
: mir
::SourceInfo
) {
340 let (index
, _
) = self.source_info_set
.insert_full(source_info
);
341 self.bcx
.set_srcloc(SourceLoc
::new(index
as u32));
344 pub(crate) fn get_caller_location(&mut self, span
: Span
) -> CValue
<'tcx
> {
345 if let Some(loc
) = self.caller_location
{
346 // `#[track_caller]` is used; return caller location instead of current location.
350 let topmost
= span
.ctxt().outer_expn().expansion_cause().unwrap_or(span
);
351 let caller
= self.tcx
.sess
.source_map().lookup_char_pos(topmost
.lo());
352 let const_loc
= self.tcx
.const_caller_location((
353 rustc_span
::symbol
::Symbol
::intern(
354 &caller
.file
.name
.prefer_remapped().to_string_lossy(),
357 caller
.col_display
as u32 + 1,
359 crate::constant
::codegen_const_value(self, const_loc
, self.tcx
.caller_location_ty())
362 pub(crate) fn anonymous_str(&mut self, msg
: &str) -> Value
{
363 let mut data_ctx
= DataContext
::new();
364 data_ctx
.define(msg
.as_bytes().to_vec().into_boxed_slice());
365 let msg_id
= self.module
.declare_anonymous_data(false, false).unwrap();
367 // Ignore DuplicateDefinition error, as the data will be the same
368 let _
= self.module
.define_data(msg_id
, &data_ctx
);
370 let local_msg_id
= self.module
.declare_data_in_func(msg_id
, self.bcx
.func
);
371 if self.clif_comments
.enabled() {
372 self.add_comment(local_msg_id
, msg
);
374 self.bcx
.ins().global_value(self.pointer_type
, local_msg_id
)
378 pub(crate) struct RevealAllLayoutCx
<'tcx
>(pub(crate) TyCtxt
<'tcx
>);
380 impl<'tcx
> LayoutOfHelpers
<'tcx
> for RevealAllLayoutCx
<'tcx
> {
381 type LayoutOfResult
= TyAndLayout
<'tcx
>;
384 fn handle_layout_err(&self, err
: LayoutError
<'tcx
>, span
: Span
, ty
: Ty
<'tcx
>) -> ! {
385 if let layout
::LayoutError
::SizeOverflow(_
) = err
{
386 self.0.sess
.span_fatal(span
, &err
.to_string())
388 span_bug
!(span
, "failed to get layout for `{}`: {}", ty
, err
)
393 impl<'tcx
> FnAbiOfHelpers
<'tcx
> for RevealAllLayoutCx
<'tcx
> {
394 type FnAbiOfResult
= &'tcx FnAbi
<'tcx
, Ty
<'tcx
>>;
397 fn handle_fn_abi_err(
399 err
: FnAbiError
<'tcx
>,
401 fn_abi_request
: FnAbiRequest
<'tcx
>,
403 if let FnAbiError
::Layout(LayoutError
::SizeOverflow(_
)) = err
{
404 self.0.sess
.span_fatal(span
, &err
.to_string())
406 match fn_abi_request
{
407 FnAbiRequest
::OfFnPtr { sig, extra_args }
=> {
410 "`fn_abi_of_fn_ptr({}, {:?})` failed: {}",
416 FnAbiRequest
::OfInstance { instance, extra_args }
=> {
419 "`fn_abi_of_instance({}, {:?})` failed: {}",
430 impl<'tcx
> layout
::HasTyCtxt
<'tcx
> for RevealAllLayoutCx
<'tcx
> {
431 fn tcx
<'b
>(&'b
self) -> TyCtxt
<'tcx
> {
436 impl<'tcx
> rustc_target
::abi
::HasDataLayout
for RevealAllLayoutCx
<'tcx
> {
437 fn data_layout(&self) -> &rustc_target
::abi
::TargetDataLayout
{
442 impl<'tcx
> layout
::HasParamEnv
<'tcx
> for RevealAllLayoutCx
<'tcx
> {
443 fn param_env(&self) -> ParamEnv
<'tcx
> {
444 ParamEnv
::reveal_all()
448 impl<'tcx
> HasTargetSpec
for RevealAllLayoutCx
<'tcx
> {
449 fn target_spec(&self) -> &Target
{