[rustc.git] / compiler / rustc_codegen_cranelift / src / num.rs

//! Various operations on integer and floating-point numbers

use crate::prelude::*;

pub(crate) fn bin_op_to_intcc(bin_op: BinOp, signed: bool) -> Option<IntCC> {
    use BinOp::*;
    use IntCC::*;
    Some(match bin_op {
        Eq => Equal,
        Lt => {
            if signed {
                SignedLessThan
            } else {
                UnsignedLessThan
            }
        }
        Le => {
            if signed {
                SignedLessThanOrEqual
            } else {
                UnsignedLessThanOrEqual
            }
        }
        Ne => NotEqual,
        Ge => {
            if signed {
                SignedGreaterThanOrEqual
            } else {
                UnsignedGreaterThanOrEqual
            }
        }
        Gt => {
            if signed {
                SignedGreaterThan
            } else {
                UnsignedGreaterThan
            }
        }
        _ => return None,
    })
}

fn codegen_compare_bin_op<'tcx>(
    fx: &mut FunctionCx<'_, '_, 'tcx>,
    bin_op: BinOp,
    signed: bool,
    lhs: Value,
    rhs: Value,
) -> CValue<'tcx> {
    let intcc = crate::num::bin_op_to_intcc(bin_op, signed).unwrap();
    let val = fx.bcx.ins().icmp(intcc, lhs, rhs);
    let val = fx.bcx.ins().bint(types::I8, val);
    CValue::by_val(val, fx.layout_of(fx.tcx.types.bool))
}

pub(crate) fn codegen_binop<'tcx>(
    fx: &mut FunctionCx<'_, '_, 'tcx>,
    bin_op: BinOp,
    in_lhs: CValue<'tcx>,
    in_rhs: CValue<'tcx>,
) -> CValue<'tcx> {
    match bin_op {
        BinOp::Eq | BinOp::Lt | BinOp::Le | BinOp::Ne | BinOp::Ge | BinOp::Gt => {
            match in_lhs.layout().ty.kind() {
                ty::Bool | ty::Uint(_) | ty::Int(_) | ty::Char => {
                    let signed = type_sign(in_lhs.layout().ty);
                    let lhs = in_lhs.load_scalar(fx);
                    let rhs = in_rhs.load_scalar(fx);

                    let (lhs, rhs) = if (bin_op == BinOp::Eq || bin_op == BinOp::Ne)
                        && (in_lhs.layout().ty.kind() == fx.tcx.types.i8.kind()
                            || in_lhs.layout().ty.kind() == fx.tcx.types.i16.kind())
                    {
                        // FIXME(CraneStation/cranelift#896) icmp_imm.i8/i16 with eq/ne for signed ints is implemented wrong.
                        (
                            fx.bcx.ins().sextend(types::I32, lhs),
                            fx.bcx.ins().sextend(types::I32, rhs),
                        )
                    } else {
                        (lhs, rhs)
                    };

                    return codegen_compare_bin_op(fx, bin_op, signed, lhs, rhs);
                }
                _ => {}
            }
        }
        _ => {}
    }

    match in_lhs.layout().ty.kind() {
        ty::Bool => crate::num::codegen_bool_binop(fx, bin_op, in_lhs, in_rhs),
        ty::Uint(_) | ty::Int(_) => crate::num::codegen_int_binop(fx, bin_op, in_lhs, in_rhs),
        ty::Float(_) => crate::num::codegen_float_binop(fx, bin_op, in_lhs, in_rhs),
        ty::RawPtr(..) | ty::FnPtr(..) => crate::num::codegen_ptr_binop(fx, bin_op, in_lhs, in_rhs),
        _ => unreachable!("{:?}({:?}, {:?})", bin_op, in_lhs.layout().ty, in_rhs.layout().ty),
    }
}

pub(crate) fn codegen_bool_binop<'tcx>(
    fx: &mut FunctionCx<'_, '_, 'tcx>,
    bin_op: BinOp,
    in_lhs: CValue<'tcx>,
    in_rhs: CValue<'tcx>,
) -> CValue<'tcx> {
    let lhs = in_lhs.load_scalar(fx);
    let rhs = in_rhs.load_scalar(fx);

    let b = fx.bcx.ins();
    let res = match bin_op {
        BinOp::BitXor => b.bxor(lhs, rhs),
        BinOp::BitAnd => b.band(lhs, rhs),
        BinOp::BitOr => b.bor(lhs, rhs),
        // Compare binops handles by `codegen_binop`.
        _ => unreachable!("{:?}({:?}, {:?})", bin_op, in_lhs, in_rhs),
    };

    CValue::by_val(res, fx.layout_of(fx.tcx.types.bool))
}

pub(crate) fn codegen_int_binop<'tcx>(
    fx: &mut FunctionCx<'_, '_, 'tcx>,
    bin_op: BinOp,
    in_lhs: CValue<'tcx>,
    in_rhs: CValue<'tcx>,
) -> CValue<'tcx> {
    if bin_op != BinOp::Shl && bin_op != BinOp::Shr {
        assert_eq!(
            in_lhs.layout().ty,
            in_rhs.layout().ty,
            "int binop requires lhs and rhs of same type"
        );
    }

    if let Some(res) = crate::codegen_i128::maybe_codegen(fx, bin_op, false, in_lhs, in_rhs) {
        return res;
    }

    let signed = type_sign(in_lhs.layout().ty);

    let lhs = in_lhs.load_scalar(fx);
    let rhs = in_rhs.load_scalar(fx);

    let b = fx.bcx.ins();
    let val = match bin_op {
        BinOp::Add => b.iadd(lhs, rhs),
        BinOp::Sub => b.isub(lhs, rhs),
        BinOp::Mul => b.imul(lhs, rhs),
        BinOp::Div => {
            if signed {
                b.sdiv(lhs, rhs)
            } else {
                b.udiv(lhs, rhs)
            }
        }
        BinOp::Rem => {
            if signed {
                b.srem(lhs, rhs)
            } else {
                b.urem(lhs, rhs)
            }
        }
        BinOp::BitXor => b.bxor(lhs, rhs),
        BinOp::BitAnd => b.band(lhs, rhs),
        BinOp::BitOr => b.bor(lhs, rhs),
        BinOp::Shl => {
            let lhs_ty = fx.bcx.func.dfg.value_type(lhs);
            let actual_shift = fx.bcx.ins().band_imm(rhs, i64::from(lhs_ty.bits() - 1));
            fx.bcx.ins().ishl(lhs, actual_shift)
        }
        BinOp::Shr => {
            let lhs_ty = fx.bcx.func.dfg.value_type(lhs);
            let actual_shift = fx.bcx.ins().band_imm(rhs, i64::from(lhs_ty.bits() - 1));
            if signed {
                fx.bcx.ins().sshr(lhs, actual_shift)
            } else {
                fx.bcx.ins().ushr(lhs, actual_shift)
            }
        }
        // Compare binops handles by `codegen_binop`.
        _ => unreachable!("{:?}({:?}, {:?})", bin_op, in_lhs.layout().ty, in_rhs.layout().ty),
    };

    CValue::by_val(val, in_lhs.layout())
}

pub(crate) fn codegen_checked_int_binop<'tcx>(
    fx: &mut FunctionCx<'_, '_, 'tcx>,
    bin_op: BinOp,
    in_lhs: CValue<'tcx>,
    in_rhs: CValue<'tcx>,
) -> CValue<'tcx> {
    if bin_op != BinOp::Shl && bin_op != BinOp::Shr {
        assert_eq!(
            in_lhs.layout().ty,
            in_rhs.layout().ty,
            "checked int binop requires lhs and rhs of same type"
        );
    }

    let lhs = in_lhs.load_scalar(fx);
    let rhs = in_rhs.load_scalar(fx);

    if let Some(res) = crate::codegen_i128::maybe_codegen(fx, bin_op, true, in_lhs, in_rhs) {
        return res;
    }

    let signed = type_sign(in_lhs.layout().ty);

    let (res, has_overflow) = match bin_op {
        BinOp::Add => {
            /*let (val, c_out) = fx.bcx.ins().iadd_cout(lhs, rhs);
            (val, c_out)*/
            // FIXME(CraneStation/cranelift#849) legalize iadd_cout for i8 and i16
            let val = fx.bcx.ins().iadd(lhs, rhs);
            let has_overflow = if !signed {
                fx.bcx.ins().icmp(IntCC::UnsignedLessThan, val, lhs)
            } else {
                let rhs_is_negative = fx.bcx.ins().icmp_imm(IntCC::SignedLessThan, rhs, 0);
                let slt = fx.bcx.ins().icmp(IntCC::SignedLessThan, val, lhs);
                fx.bcx.ins().bxor(rhs_is_negative, slt)
            };
            (val, has_overflow)
        }
        BinOp::Sub => {
            /*let (val, b_out) = fx.bcx.ins().isub_bout(lhs, rhs);
            (val, b_out)*/
            // FIXME(CraneStation/cranelift#849) legalize isub_bout for i8 and i16
            let val = fx.bcx.ins().isub(lhs, rhs);
            let has_overflow = if !signed {
                fx.bcx.ins().icmp(IntCC::UnsignedGreaterThan, val, lhs)
            } else {
                let rhs_is_negative = fx.bcx.ins().icmp_imm(IntCC::SignedLessThan, rhs, 0);
                let sgt = fx.bcx.ins().icmp(IntCC::SignedGreaterThan, val, lhs);
                fx.bcx.ins().bxor(rhs_is_negative, sgt)
            };
            (val, has_overflow)
        }
        BinOp::Mul => {
            let ty = fx.bcx.func.dfg.value_type(lhs);
            match ty {
                types::I8 | types::I16 | types::I32 if !signed => {
                    let lhs = fx.bcx.ins().uextend(ty.double_width().unwrap(), lhs);
                    let rhs = fx.bcx.ins().uextend(ty.double_width().unwrap(), rhs);
                    let val = fx.bcx.ins().imul(lhs, rhs);
                    let has_overflow = fx.bcx.ins().icmp_imm(
                        IntCC::UnsignedGreaterThan,
                        val,
                        (1 << ty.bits()) - 1,
                    );
                    let val = fx.bcx.ins().ireduce(ty, val);
                    (val, has_overflow)
                }
                types::I8 | types::I16 | types::I32 if signed => {
                    let lhs = fx.bcx.ins().sextend(ty.double_width().unwrap(), lhs);
                    let rhs = fx.bcx.ins().sextend(ty.double_width().unwrap(), rhs);
                    let val = fx.bcx.ins().imul(lhs, rhs);
                    let has_underflow =
                        fx.bcx.ins().icmp_imm(IntCC::SignedLessThan, val, -(1 << (ty.bits() - 1)));
                    let has_overflow = fx.bcx.ins().icmp_imm(
                        IntCC::SignedGreaterThan,
                        val,
                        (1 << (ty.bits() - 1)) - 1,
                    );
                    let val = fx.bcx.ins().ireduce(ty, val);
                    (val, fx.bcx.ins().bor(has_underflow, has_overflow))
                }
                types::I64 => {
                    let val = fx.bcx.ins().imul(lhs, rhs);
                    let has_overflow = if !signed {
                        let val_hi = fx.bcx.ins().umulhi(lhs, rhs);
                        fx.bcx.ins().icmp_imm(IntCC::NotEqual, val_hi, 0)
                    } else {
                        let val_hi = fx.bcx.ins().smulhi(lhs, rhs);
                        let not_all_zero = fx.bcx.ins().icmp_imm(IntCC::NotEqual, val_hi, 0);
                        let not_all_ones = fx.bcx.ins().icmp_imm(
                            IntCC::NotEqual,
                            val_hi,
                            u64::try_from((1u128 << ty.bits()) - 1).unwrap() as i64,
                        );
                        fx.bcx.ins().band(not_all_zero, not_all_ones)
                    };
                    (val, has_overflow)
                }
                types::I128 => {
                    unreachable!("i128 should have been handled by codegen_i128::maybe_codegen")
                }
                _ => unreachable!("invalid non-integer type {}", ty),
            }
        }
        BinOp::Shl => {
            let lhs_ty = fx.bcx.func.dfg.value_type(lhs);
            let actual_shift = fx.bcx.ins().band_imm(rhs, i64::from(lhs_ty.bits() - 1));
            let actual_shift = clif_intcast(fx, actual_shift, types::I8, false);
            let val = fx.bcx.ins().ishl(lhs, actual_shift);
            let ty = fx.bcx.func.dfg.value_type(val);
            let max_shift = i64::from(ty.bits()) - 1;
            let has_overflow = fx.bcx.ins().icmp_imm(IntCC::UnsignedGreaterThan, rhs, max_shift);
            (val, has_overflow)
        }
        BinOp::Shr => {
            let lhs_ty = fx.bcx.func.dfg.value_type(lhs);
            let actual_shift = fx.bcx.ins().band_imm(rhs, i64::from(lhs_ty.bits() - 1));
            let actual_shift = clif_intcast(fx, actual_shift, types::I8, false);
            let val = if !signed {
                fx.bcx.ins().ushr(lhs, actual_shift)
            } else {
                fx.bcx.ins().sshr(lhs, actual_shift)
            };
            let ty = fx.bcx.func.dfg.value_type(val);
            let max_shift = i64::from(ty.bits()) - 1;
            let has_overflow = fx.bcx.ins().icmp_imm(IntCC::UnsignedGreaterThan, rhs, max_shift);
            (val, has_overflow)
        }
        _ => bug!("binop {:?} on checked int/uint lhs: {:?} rhs: {:?}", bin_op, in_lhs, in_rhs),
    };

    let has_overflow = fx.bcx.ins().bint(types::I8, has_overflow);

    let out_layout = fx.layout_of(fx.tcx.mk_tup([in_lhs.layout().ty, fx.tcx.types.bool].iter()));
    CValue::by_val_pair(res, has_overflow, out_layout)
}

pub(crate) fn codegen_float_binop<'tcx>(
    fx: &mut FunctionCx<'_, '_, 'tcx>,
    bin_op: BinOp,
    in_lhs: CValue<'tcx>,
    in_rhs: CValue<'tcx>,
) -> CValue<'tcx> {
    assert_eq!(in_lhs.layout().ty, in_rhs.layout().ty);

    let lhs = in_lhs.load_scalar(fx);
    let rhs = in_rhs.load_scalar(fx);

    let b = fx.bcx.ins();
    let res = match bin_op {
        BinOp::Add => b.fadd(lhs, rhs),
        BinOp::Sub => b.fsub(lhs, rhs),
        BinOp::Mul => b.fmul(lhs, rhs),
        BinOp::Div => b.fdiv(lhs, rhs),
        BinOp::Rem => {
            let name = match in_lhs.layout().ty.kind() {
                ty::Float(FloatTy::F32) => "fmodf",
                ty::Float(FloatTy::F64) => "fmod",
                _ => bug!(),
            };
            return fx.easy_call(name, &[in_lhs, in_rhs], in_lhs.layout().ty);
        }
        BinOp::Eq | BinOp::Lt | BinOp::Le | BinOp::Ne | BinOp::Ge | BinOp::Gt => {
            let fltcc = match bin_op {
                BinOp::Eq => FloatCC::Equal,
                BinOp::Lt => FloatCC::LessThan,
                BinOp::Le => FloatCC::LessThanOrEqual,
                BinOp::Ne => FloatCC::NotEqual,
                BinOp::Ge => FloatCC::GreaterThanOrEqual,
                BinOp::Gt => FloatCC::GreaterThan,
                _ => unreachable!(),
            };
            let val = fx.bcx.ins().fcmp(fltcc, lhs, rhs);
            let val = fx.bcx.ins().bint(types::I8, val);
            return CValue::by_val(val, fx.layout_of(fx.tcx.types.bool));
        }
        _ => unreachable!("{:?}({:?}, {:?})", bin_op, in_lhs, in_rhs),
    };

    CValue::by_val(res, in_lhs.layout())
}

pub(crate) fn codegen_ptr_binop<'tcx>(
    fx: &mut FunctionCx<'_, '_, 'tcx>,
    bin_op: BinOp,
    in_lhs: CValue<'tcx>,
    in_rhs: CValue<'tcx>,
) -> CValue<'tcx> {
    let is_thin_ptr = in_lhs
        .layout()
        .ty
        .builtin_deref(true)
        .map(|TypeAndMut { ty, mutbl: _ }| !has_ptr_meta(fx.tcx, ty))
        .unwrap_or(true);

    if is_thin_ptr {
        match bin_op {
            BinOp::Eq | BinOp::Lt | BinOp::Le | BinOp::Ne | BinOp::Ge | BinOp::Gt => {
                let lhs = in_lhs.load_scalar(fx);
                let rhs = in_rhs.load_scalar(fx);

                codegen_compare_bin_op(fx, bin_op, false, lhs, rhs)
            }
            BinOp::Offset => {
                let pointee_ty = in_lhs.layout().ty.builtin_deref(true).unwrap().ty;
                let (base, offset) = (in_lhs, in_rhs.load_scalar(fx));
                let pointee_size = fx.layout_of(pointee_ty).size.bytes();
                let ptr_diff = fx.bcx.ins().imul_imm(offset, pointee_size as i64);
                let base_val = base.load_scalar(fx);
                let res = fx.bcx.ins().iadd(base_val, ptr_diff);
                CValue::by_val(res, base.layout())
            }
            _ => unreachable!("{:?}({:?}, {:?})", bin_op, in_lhs, in_rhs),
        }
    } else {
        let (lhs_ptr, lhs_extra) = in_lhs.load_scalar_pair(fx);
        let (rhs_ptr, rhs_extra) = in_rhs.load_scalar_pair(fx);

        let res = match bin_op {
            BinOp::Eq => {
                let ptr_eq = fx.bcx.ins().icmp(IntCC::Equal, lhs_ptr, rhs_ptr);
                let extra_eq = fx.bcx.ins().icmp(IntCC::Equal, lhs_extra, rhs_extra);
                fx.bcx.ins().band(ptr_eq, extra_eq)
            }
            BinOp::Ne => {
                let ptr_ne = fx.bcx.ins().icmp(IntCC::NotEqual, lhs_ptr, rhs_ptr);
                let extra_ne = fx.bcx.ins().icmp(IntCC::NotEqual, lhs_extra, rhs_extra);
                fx.bcx.ins().bor(ptr_ne, extra_ne)
            }
            BinOp::Lt | BinOp::Le | BinOp::Ge | BinOp::Gt => {
                let ptr_eq = fx.bcx.ins().icmp(IntCC::Equal, lhs_ptr, rhs_ptr);

                let ptr_cmp =
                    fx.bcx.ins().icmp(bin_op_to_intcc(bin_op, false).unwrap(), lhs_ptr, rhs_ptr);
                let extra_cmp = fx.bcx.ins().icmp(
                    bin_op_to_intcc(bin_op, false).unwrap(),
                    lhs_extra,
                    rhs_extra,
                );

                fx.bcx.ins().select(ptr_eq, extra_cmp, ptr_cmp)
            }
            _ => panic!("bin_op {:?} on ptr", bin_op),
        };

        CValue::by_val(fx.bcx.ins().bint(types::I8, res), fx.layout_of(fx.tcx.types.bool))
    }
}
Commit	Line	Data
29967ef6 XL	1	//! Various operations on integer and floating-point numbers
	2
	3	use crate::prelude::*;
	4
	5	pub(crate) fn bin_op_to_intcc(bin_op: BinOp, signed: bool) -> Option<IntCC> {
	6	use BinOp::*;
	7	use IntCC::*;
	8	Some(match bin_op {
	9	Eq => Equal,
	10	Lt => {
	11	if signed {
	12	SignedLessThan
	13	} else {
	14	UnsignedLessThan
	15	}
	16	}
	17	Le => {
	18	if signed {
	19	SignedLessThanOrEqual
	20	} else {
	21	UnsignedLessThanOrEqual
	22	}
	23	}
	24	Ne => NotEqual,
	25	Ge => {
	26	if signed {
	27	SignedGreaterThanOrEqual
	28	} else {
	29	UnsignedGreaterThanOrEqual
	30	}
	31	}
	32	Gt => {
	33	if signed {
	34	SignedGreaterThan
	35	} else {
	36	UnsignedGreaterThan
	37	}
	38	}
	39	_ => return None,
	40	})
	41	}
	42
	43	fn codegen_compare_bin_op<'tcx>(
6a06907d	44	fx: &mut FunctionCx<'_, '_, 'tcx>,
29967ef6 XL	45	bin_op: BinOp,
	46	signed: bool,
	47	lhs: Value,
	48	rhs: Value,
	49	) -> CValue<'tcx> {
	50	let intcc = crate::num::bin_op_to_intcc(bin_op, signed).unwrap();
	51	let val = fx.bcx.ins().icmp(intcc, lhs, rhs);
	52	let val = fx.bcx.ins().bint(types::I8, val);
	53	CValue::by_val(val, fx.layout_of(fx.tcx.types.bool))
	54	}
	55
	56	pub(crate) fn codegen_binop<'tcx>(
6a06907d	57	fx: &mut FunctionCx<'_, '_, 'tcx>,
29967ef6 XL	58	bin_op: BinOp,
	59	in_lhs: CValue<'tcx>,
	60	in_rhs: CValue<'tcx>,
	61	) -> CValue<'tcx> {
	62	match bin_op {
	63	BinOp::Eq \| BinOp::Lt \| BinOp::Le \| BinOp::Ne \| BinOp::Ge \| BinOp::Gt => {
	64	match in_lhs.layout().ty.kind() {
	65	ty::Bool \| ty::Uint(_) \| ty::Int(_) \| ty::Char => {
	66	let signed = type_sign(in_lhs.layout().ty);
	67	let lhs = in_lhs.load_scalar(fx);
	68	let rhs = in_rhs.load_scalar(fx);
	69
	70	let (lhs, rhs) = if (bin_op == BinOp::Eq \|\| bin_op == BinOp::Ne)
	71	&& (in_lhs.layout().ty.kind() == fx.tcx.types.i8.kind()
	72	\|\| in_lhs.layout().ty.kind() == fx.tcx.types.i16.kind())
	73	{
	74	// FIXME(CraneStation/cranelift#896) icmp_imm.i8/i16 with eq/ne for signed ints is implemented wrong.
	75	(
	76	fx.bcx.ins().sextend(types::I32, lhs),
	77	fx.bcx.ins().sextend(types::I32, rhs),
	78	)
	79	} else {
	80	(lhs, rhs)
	81	};
	82
	83	return codegen_compare_bin_op(fx, bin_op, signed, lhs, rhs);
	84	}
	85	_ => {}
	86	}
	87	}
	88	_ => {}
	89	}
	90
	91	match in_lhs.layout().ty.kind() {
	92	ty::Bool => crate::num::codegen_bool_binop(fx, bin_op, in_lhs, in_rhs),
	93	ty::Uint(_) \| ty::Int(_) => crate::num::codegen_int_binop(fx, bin_op, in_lhs, in_rhs),
	94	ty::Float(_) => crate::num::codegen_float_binop(fx, bin_op, in_lhs, in_rhs),
	95	ty::RawPtr(..) \| ty::FnPtr(..) => crate::num::codegen_ptr_binop(fx, bin_op, in_lhs, in_rhs),
6a06907d	96	_ => unreachable!("{:?}({:?}, {:?})", bin_op, in_lhs.layout().ty, in_rhs.layout().ty),
29967ef6 XL	97	}
	98	}
	99
	100	pub(crate) fn codegen_bool_binop<'tcx>(
6a06907d	101	fx: &mut FunctionCx<'_, '_, 'tcx>,
29967ef6 XL	102	bin_op: BinOp,
	103	in_lhs: CValue<'tcx>,
	104	in_rhs: CValue<'tcx>,
	105	) -> CValue<'tcx> {
	106	let lhs = in_lhs.load_scalar(fx);
	107	let rhs = in_rhs.load_scalar(fx);
	108
	109	let b = fx.bcx.ins();
	110	let res = match bin_op {
	111	BinOp::BitXor => b.bxor(lhs, rhs),
	112	BinOp::BitAnd => b.band(lhs, rhs),
	113	BinOp::BitOr => b.bor(lhs, rhs),
	114	// Compare binops handles by `codegen_binop`.
	115	_ => unreachable!("{:?}({:?}, {:?})", bin_op, in_lhs, in_rhs),
	116	};
	117
	118	CValue::by_val(res, fx.layout_of(fx.tcx.types.bool))
	119	}
	120
	121	pub(crate) fn codegen_int_binop<'tcx>(
6a06907d	122	fx: &mut FunctionCx<'_, '_, 'tcx>,
29967ef6 XL	123	bin_op: BinOp,
	124	in_lhs: CValue<'tcx>,
	125	in_rhs: CValue<'tcx>,
	126	) -> CValue<'tcx> {
	127	if bin_op != BinOp::Shl && bin_op != BinOp::Shr {
	128	assert_eq!(
	129	in_lhs.layout().ty,
	130	in_rhs.layout().ty,
	131	"int binop requires lhs and rhs of same type"
	132	);
	133	}
	134
	135	if let Some(res) = crate::codegen_i128::maybe_codegen(fx, bin_op, false, in_lhs, in_rhs) {
	136	return res;
	137	}
	138
	139	let signed = type_sign(in_lhs.layout().ty);
	140
	141	let lhs = in_lhs.load_scalar(fx);
	142	let rhs = in_rhs.load_scalar(fx);
	143
	144	let b = fx.bcx.ins();
	145	let val = match bin_op {
	146	BinOp::Add => b.iadd(lhs, rhs),
	147	BinOp::Sub => b.isub(lhs, rhs),
	148	BinOp::Mul => b.imul(lhs, rhs),
	149	BinOp::Div => {
	150	if signed {
	151	b.sdiv(lhs, rhs)
	152	} else {
	153	b.udiv(lhs, rhs)
	154	}
	155	}
	156	BinOp::Rem => {
	157	if signed {
	158	b.srem(lhs, rhs)
	159	} else {
	160	b.urem(lhs, rhs)
	161	}
	162	}
	163	BinOp::BitXor => b.bxor(lhs, rhs),
	164	BinOp::BitAnd => b.band(lhs, rhs),
	165	BinOp::BitOr => b.bor(lhs, rhs),
	166	BinOp::Shl => {
	167	let lhs_ty = fx.bcx.func.dfg.value_type(lhs);
	168	let actual_shift = fx.bcx.ins().band_imm(rhs, i64::from(lhs_ty.bits() - 1));
29967ef6 XL	169	fx.bcx.ins().ishl(lhs, actual_shift)
	170	}
	171	BinOp::Shr => {
	172	let lhs_ty = fx.bcx.func.dfg.value_type(lhs);
	173	let actual_shift = fx.bcx.ins().band_imm(rhs, i64::from(lhs_ty.bits() - 1));
29967ef6 XL	174	if signed {
	175	fx.bcx.ins().sshr(lhs, actual_shift)
	176	} else {
	177	fx.bcx.ins().ushr(lhs, actual_shift)
	178	}
	179	}
	180	// Compare binops handles by `codegen_binop`.
6a06907d	181	_ => unreachable!("{:?}({:?}, {:?})", bin_op, in_lhs.layout().ty, in_rhs.layout().ty),
29967ef6 XL	182	};
	183
	184	CValue::by_val(val, in_lhs.layout())
	185	}
	186
	187	pub(crate) fn codegen_checked_int_binop<'tcx>(
6a06907d	188	fx: &mut FunctionCx<'_, '_, 'tcx>,
29967ef6 XL	189	bin_op: BinOp,
	190	in_lhs: CValue<'tcx>,
	191	in_rhs: CValue<'tcx>,
	192	) -> CValue<'tcx> {
	193	if bin_op != BinOp::Shl && bin_op != BinOp::Shr {
	194	assert_eq!(
	195	in_lhs.layout().ty,
	196	in_rhs.layout().ty,
	197	"checked int binop requires lhs and rhs of same type"
	198	);
	199	}
	200
	201	let lhs = in_lhs.load_scalar(fx);
	202	let rhs = in_rhs.load_scalar(fx);
	203
	204	if let Some(res) = crate::codegen_i128::maybe_codegen(fx, bin_op, true, in_lhs, in_rhs) {
	205	return res;
	206	}
	207
	208	let signed = type_sign(in_lhs.layout().ty);
	209
	210	let (res, has_overflow) = match bin_op {
	211	BinOp::Add => {
	212	/*let (val, c_out) = fx.bcx.ins().iadd_cout(lhs, rhs);
	213	(val, c_out)*/
	214	// FIXME(CraneStation/cranelift#849) legalize iadd_cout for i8 and i16
	215	let val = fx.bcx.ins().iadd(lhs, rhs);
	216	let has_overflow = if !signed {
	217	fx.bcx.ins().icmp(IntCC::UnsignedLessThan, val, lhs)
	218	} else {
	219	let rhs_is_negative = fx.bcx.ins().icmp_imm(IntCC::SignedLessThan, rhs, 0);
	220	let slt = fx.bcx.ins().icmp(IntCC::SignedLessThan, val, lhs);
	221	fx.bcx.ins().bxor(rhs_is_negative, slt)
	222	};
	223	(val, has_overflow)
	224	}
	225	BinOp::Sub => {
	226	/*let (val, b_out) = fx.bcx.ins().isub_bout(lhs, rhs);
	227	(val, b_out)*/
	228	// FIXME(CraneStation/cranelift#849) legalize isub_bout for i8 and i16
	229	let val = fx.bcx.ins().isub(lhs, rhs);
	230	let has_overflow = if !signed {
	231	fx.bcx.ins().icmp(IntCC::UnsignedGreaterThan, val, lhs)
	232	} else {
	233	let rhs_is_negative = fx.bcx.ins().icmp_imm(IntCC::SignedLessThan, rhs, 0);
	234	let sgt = fx.bcx.ins().icmp(IntCC::SignedGreaterThan, val, lhs);
	235	fx.bcx.ins().bxor(rhs_is_negative, sgt)
	236	};
	237	(val, has_overflow)
	238	}
	239	BinOp::Mul => {
	240	let ty = fx.bcx.func.dfg.value_type(lhs);
	241	match ty {
	242	types::I8 \| types::I16 \| types::I32 if !signed => {
	243	let lhs = fx.bcx.ins().uextend(ty.double_width().unwrap(), lhs);
	244	let rhs = fx.bcx.ins().uextend(ty.double_width().unwrap(), rhs);
	245	let val = fx.bcx.ins().imul(lhs, rhs);
	246	let has_overflow = fx.bcx.ins().icmp_imm(
	247	IntCC::UnsignedGreaterThan,
	248	val,
	249	(1 << ty.bits()) - 1,
	250	);
	251	let val = fx.bcx.ins().ireduce(ty, val);
	252	(val, has_overflow)
253	}
254	types::I8 \| types::I16 \| types::I32 if signed => {
255	let lhs = fx.bcx.ins().sextend(ty.double_width().unwrap(), lhs);
256	let rhs = fx.bcx.ins().sextend(ty.double_width().unwrap(), rhs);
257	let val = fx.bcx.ins().imul(lhs, rhs);
258	let has_underflow =
6a06907d	259	fx.bcx.ins().icmp_imm(IntCC::SignedLessThan, val, -(1 << (ty.bits() - 1)));
29967ef6 XL	260	let has_overflow = fx.bcx.ins().icmp_imm(
	261	IntCC::SignedGreaterThan,
	262	val,
	263	(1 << (ty.bits() - 1)) - 1,
	264	);
	265	let val = fx.bcx.ins().ireduce(ty, val);
	266	(val, fx.bcx.ins().bor(has_underflow, has_overflow))
	267	}
	268	types::I64 => {
29967ef6 XL	269	let val = fx.bcx.ins().imul(lhs, rhs);
	270	let has_overflow = if !signed {
	271	let val_hi = fx.bcx.ins().umulhi(lhs, rhs);
	272	fx.bcx.ins().icmp_imm(IntCC::NotEqual, val_hi, 0)
	273	} else {
	274	let val_hi = fx.bcx.ins().smulhi(lhs, rhs);
	275	let not_all_zero = fx.bcx.ins().icmp_imm(IntCC::NotEqual, val_hi, 0);
	276	let not_all_ones = fx.bcx.ins().icmp_imm(
	277	IntCC::NotEqual,
	278	val_hi,
	279	u64::try_from((1u128 << ty.bits()) - 1).unwrap() as i64,
	280	);
	281	fx.bcx.ins().band(not_all_zero, not_all_ones)
	282	};
	283	(val, has_overflow)
	284	}
	285	types::I128 => {
	286	unreachable!("i128 should have been handled by codegen_i128::maybe_codegen")
	287	}
	288	_ => unreachable!("invalid non-integer type {}", ty),
	289	}
	290	}
	291	BinOp::Shl => {
	292	let lhs_ty = fx.bcx.func.dfg.value_type(lhs);
	293	let actual_shift = fx.bcx.ins().band_imm(rhs, i64::from(lhs_ty.bits() - 1));
	294	let actual_shift = clif_intcast(fx, actual_shift, types::I8, false);
	295	let val = fx.bcx.ins().ishl(lhs, actual_shift);
	296	let ty = fx.bcx.func.dfg.value_type(val);
	297	let max_shift = i64::from(ty.bits()) - 1;
6a06907d	298	let has_overflow = fx.bcx.ins().icmp_imm(IntCC::UnsignedGreaterThan, rhs, max_shift);
29967ef6 XL	299	(val, has_overflow)
	300	}
	301	BinOp::Shr => {
	302	let lhs_ty = fx.bcx.func.dfg.value_type(lhs);
	303	let actual_shift = fx.bcx.ins().band_imm(rhs, i64::from(lhs_ty.bits() - 1));
	304	let actual_shift = clif_intcast(fx, actual_shift, types::I8, false);
	305	let val = if !signed {
	306	fx.bcx.ins().ushr(lhs, actual_shift)
	307	} else {
	308	fx.bcx.ins().sshr(lhs, actual_shift)
	309	};
	310	let ty = fx.bcx.func.dfg.value_type(val);
	311	let max_shift = i64::from(ty.bits()) - 1;
6a06907d	312	let has_overflow = fx.bcx.ins().icmp_imm(IntCC::UnsignedGreaterThan, rhs, max_shift);
29967ef6 XL	313	(val, has_overflow)
29967ef6 XL	314	}
6a06907d	315	_ => bug!("binop {:?} on checked int/uint lhs: {:?} rhs: {:?}", bin_op, in_lhs, in_rhs),
29967ef6 XL	316	};
	317
	318	let has_overflow = fx.bcx.ins().bint(types::I8, has_overflow);
	319
6a06907d XL	320	let out_layout = fx.layout_of(fx.tcx.mk_tup([in_lhs.layout().ty, fx.tcx.types.bool].iter()));
6a06907d XL	321	CValue::by_val_pair(res, has_overflow, out_layout)
29967ef6 XL	322	}
	323
	324	pub(crate) fn codegen_float_binop<'tcx>(
6a06907d	325	fx: &mut FunctionCx<'_, '_, 'tcx>,
29967ef6 XL	326	bin_op: BinOp,
	327	in_lhs: CValue<'tcx>,
	328	in_rhs: CValue<'tcx>,
	329	) -> CValue<'tcx> {
	330	assert_eq!(in_lhs.layout().ty, in_rhs.layout().ty);
	331
	332	let lhs = in_lhs.load_scalar(fx);
	333	let rhs = in_rhs.load_scalar(fx);
	334
	335	let b = fx.bcx.ins();
	336	let res = match bin_op {
	337	BinOp::Add => b.fadd(lhs, rhs),
	338	BinOp::Sub => b.fsub(lhs, rhs),
	339	BinOp::Mul => b.fmul(lhs, rhs),
	340	BinOp::Div => b.fdiv(lhs, rhs),
	341	BinOp::Rem => {
	342	let name = match in_lhs.layout().ty.kind() {
	343	ty::Float(FloatTy::F32) => "fmodf",
	344	ty::Float(FloatTy::F64) => "fmod",
	345	_ => bug!(),
	346	};
	347	return fx.easy_call(name, &[in_lhs, in_rhs], in_lhs.layout().ty);
	348	}
	349	BinOp::Eq \| BinOp::Lt \| BinOp::Le \| BinOp::Ne \| BinOp::Ge \| BinOp::Gt => {
	350	let fltcc = match bin_op {
	351	BinOp::Eq => FloatCC::Equal,
	352	BinOp::Lt => FloatCC::LessThan,
	353	BinOp::Le => FloatCC::LessThanOrEqual,
	354	BinOp::Ne => FloatCC::NotEqual,
	355	BinOp::Ge => FloatCC::GreaterThanOrEqual,
	356	BinOp::Gt => FloatCC::GreaterThan,
	357	_ => unreachable!(),
	358	};
	359	let val = fx.bcx.ins().fcmp(fltcc, lhs, rhs);
	360	let val = fx.bcx.ins().bint(types::I8, val);
	361	return CValue::by_val(val, fx.layout_of(fx.tcx.types.bool));
	362	}
	363	_ => unreachable!("{:?}({:?}, {:?})", bin_op, in_lhs, in_rhs),
	364	};
	365
	366	CValue::by_val(res, in_lhs.layout())
	367	}
	368
	369	pub(crate) fn codegen_ptr_binop<'tcx>(
6a06907d	370	fx: &mut FunctionCx<'_, '_, 'tcx>,
29967ef6 XL	371	bin_op: BinOp,
	372	in_lhs: CValue<'tcx>,
	373	in_rhs: CValue<'tcx>,
	374	) -> CValue<'tcx> {
	375	let is_thin_ptr = in_lhs
	376	.layout()
	377	.ty
	378	.builtin_deref(true)
	379	.map(\|TypeAndMut { ty, mutbl: _ }\| !has_ptr_meta(fx.tcx, ty))
	380	.unwrap_or(true);
	381
	382	if is_thin_ptr {
	383	match bin_op {
	384	BinOp::Eq \| BinOp::Lt \| BinOp::Le \| BinOp::Ne \| BinOp::Ge \| BinOp::Gt => {
	385	let lhs = in_lhs.load_scalar(fx);
	386	let rhs = in_rhs.load_scalar(fx);
	387
cdc7bbd5	388	codegen_compare_bin_op(fx, bin_op, false, lhs, rhs)
29967ef6 XL	389	}
	390	BinOp::Offset => {
	391	let pointee_ty = in_lhs.layout().ty.builtin_deref(true).unwrap().ty;
	392	let (base, offset) = (in_lhs, in_rhs.load_scalar(fx));
	393	let pointee_size = fx.layout_of(pointee_ty).size.bytes();
	394	let ptr_diff = fx.bcx.ins().imul_imm(offset, pointee_size as i64);
	395	let base_val = base.load_scalar(fx);
	396	let res = fx.bcx.ins().iadd(base_val, ptr_diff);
cdc7bbd5	397	CValue::by_val(res, base.layout())
29967ef6 XL	398	}
29967ef6 XL	399	_ => unreachable!("{:?}({:?}, {:?})", bin_op, in_lhs, in_rhs),
cdc7bbd5	400	}
29967ef6 XL	401	} else {
	402	let (lhs_ptr, lhs_extra) = in_lhs.load_scalar_pair(fx);
	403	let (rhs_ptr, rhs_extra) = in_rhs.load_scalar_pair(fx);
	404
	405	let res = match bin_op {
	406	BinOp::Eq => {
	407	let ptr_eq = fx.bcx.ins().icmp(IntCC::Equal, lhs_ptr, rhs_ptr);
	408	let extra_eq = fx.bcx.ins().icmp(IntCC::Equal, lhs_extra, rhs_extra);
	409	fx.bcx.ins().band(ptr_eq, extra_eq)
	410	}
	411	BinOp::Ne => {
	412	let ptr_ne = fx.bcx.ins().icmp(IntCC::NotEqual, lhs_ptr, rhs_ptr);
	413	let extra_ne = fx.bcx.ins().icmp(IntCC::NotEqual, lhs_extra, rhs_extra);
	414	fx.bcx.ins().bor(ptr_ne, extra_ne)
	415	}
	416	BinOp::Lt \| BinOp::Le \| BinOp::Ge \| BinOp::Gt => {
	417	let ptr_eq = fx.bcx.ins().icmp(IntCC::Equal, lhs_ptr, rhs_ptr);
	418
	419	let ptr_cmp =
6a06907d	420	fx.bcx.ins().icmp(bin_op_to_intcc(bin_op, false).unwrap(), lhs_ptr, rhs_ptr);
29967ef6 XL	421	let extra_cmp = fx.bcx.ins().icmp(
	422	bin_op_to_intcc(bin_op, false).unwrap(),
	423	lhs_extra,
	424	rhs_extra,
	425	);
	426
	427	fx.bcx.ins().select(ptr_eq, extra_cmp, ptr_cmp)
	428	}
	429	_ => panic!("bin_op {:?} on ptr", bin_op),
	430	};
	431
6a06907d	432	CValue::by_val(fx.bcx.ins().bint(types::I8, res), fx.layout_of(fx.tcx.types.bool))
29967ef6 XL	433	}
29967ef6 XL	434	}