[rustc.git] / compiler / rustc_target / src / asm / x86.rs

use super::{InlineAsmArch, InlineAsmType};
use crate::spec::Target;
use rustc_data_structures::stable_set::FxHashSet;
use rustc_macros::HashStable_Generic;
use rustc_span::Symbol;
use std::fmt;

def_reg_class! {
    X86 X86InlineAsmRegClass {
        reg,
        reg_abcd,
        reg_byte,
        xmm_reg,
        ymm_reg,
        zmm_reg,
        kreg,
        mmx_reg,
        x87_reg,
    }
}

impl X86InlineAsmRegClass {
    pub fn valid_modifiers(self, arch: super::InlineAsmArch) -> &'static [char] {
        match self {
            Self::reg => {
                if arch == InlineAsmArch::X86_64 {
                    &['l', 'x', 'e', 'r']
                } else {
                    &['x', 'e']
                }
            }
            Self::reg_abcd => {
                if arch == InlineAsmArch::X86_64 {
                    &['l', 'h', 'x', 'e', 'r']
                } else {
                    &['l', 'h', 'x', 'e']
                }
            }
            Self::reg_byte => &[],
            Self::xmm_reg | Self::ymm_reg | Self::zmm_reg => &['x', 'y', 'z'],
            Self::kreg => &[],
            Self::mmx_reg | Self::x87_reg => &[],
        }
    }

    pub fn suggest_class(self, _arch: InlineAsmArch, ty: InlineAsmType) -> Option<Self> {
        match self {
            Self::reg | Self::reg_abcd if ty.size().bits() == 8 => Some(Self::reg_byte),
            _ => None,
        }
    }

    pub fn suggest_modifier(
        self,
        arch: InlineAsmArch,
        ty: InlineAsmType,
    ) -> Option<(char, &'static str)> {
        match self {
            Self::reg => match ty.size().bits() {
                16 => Some(('x', "ax")),
                32 if arch == InlineAsmArch::X86_64 => Some(('e', "eax")),
                _ => None,
            },
            Self::reg_abcd => match ty.size().bits() {
                16 => Some(('x', "ax")),
                32 if arch == InlineAsmArch::X86_64 => Some(('e', "eax")),
                _ => None,
            },
            Self::reg_byte => None,
            Self::xmm_reg => None,
            Self::ymm_reg => match ty.size().bits() {
                256 => None,
                _ => Some(('x', "xmm0")),
            },
            Self::zmm_reg => match ty.size().bits() {
                512 => None,
                256 => Some(('y', "ymm0")),
                _ => Some(('x', "xmm0")),
            },
            Self::kreg => None,
            Self::mmx_reg | Self::x87_reg => None,
        }
    }

    pub fn default_modifier(self, arch: InlineAsmArch) -> Option<(char, &'static str)> {
        match self {
            Self::reg | Self::reg_abcd => {
                if arch == InlineAsmArch::X86_64 {
                    Some(('r', "rax"))
                } else {
                    Some(('e', "eax"))
                }
            }
            Self::reg_byte => None,
            Self::xmm_reg => Some(('x', "xmm0")),
            Self::ymm_reg => Some(('y', "ymm0")),
            Self::zmm_reg => Some(('z', "zmm0")),
            Self::kreg => None,
            Self::mmx_reg | Self::x87_reg => None,
        }
    }

    pub fn supported_types(
        self,
        arch: InlineAsmArch,
    ) -> &'static [(InlineAsmType, Option<Symbol>)] {
        match self {
            Self::reg | Self::reg_abcd => {
                if arch == InlineAsmArch::X86_64 {
                    types! { _: I16, I32, I64, F32, F64; }
                } else {
                    types! { _: I16, I32, F32; }
                }
            }
            Self::reg_byte => types! { _: I8; },
            Self::xmm_reg => types! {
                sse: I32, I64, F32, F64,
                  VecI8(16), VecI16(8), VecI32(4), VecI64(2), VecF32(4), VecF64(2);
            },
            Self::ymm_reg => types! {
                avx: I32, I64, F32, F64,
                    VecI8(16), VecI16(8), VecI32(4), VecI64(2), VecF32(4), VecF64(2),
                    VecI8(32), VecI16(16), VecI32(8), VecI64(4), VecF32(8), VecF64(4);
            },
            Self::zmm_reg => types! {
                avx512f: I32, I64, F32, F64,
                    VecI8(16), VecI16(8), VecI32(4), VecI64(2), VecF32(4), VecF64(2),
                    VecI8(32), VecI16(16), VecI32(8), VecI64(4), VecF32(8), VecF64(4),
                    VecI8(64), VecI16(32), VecI32(16), VecI64(8), VecF32(16), VecF64(8);
            },
            Self::kreg => types! {
                avx512f: I8, I16;
                avx512bw: I32, I64;
            },
            Self::mmx_reg | Self::x87_reg => &[],
        }
    }
}

fn x86_64_only(
    arch: InlineAsmArch,
    _target_features: &FxHashSet<Symbol>,
    _target: &Target,
    _is_clobber: bool,
) -> Result<(), &'static str> {
    match arch {
        InlineAsmArch::X86 => Err("register is only available on x86_64"),
        InlineAsmArch::X86_64 => Ok(()),
        _ => unreachable!(),
    }
}

fn high_byte(
    arch: InlineAsmArch,
    _target_features: &FxHashSet<Symbol>,
    _target: &Target,
    _is_clobber: bool,
) -> Result<(), &'static str> {
    match arch {
        InlineAsmArch::X86_64 => Err("high byte registers cannot be used as an operand on x86_64"),
        _ => Ok(()),
    }
}

fn rbx_reserved(
    arch: InlineAsmArch,
    _target_features: &FxHashSet<Symbol>,
    _target: &Target,
    _is_clobber: bool,
) -> Result<(), &'static str> {
    match arch {
        InlineAsmArch::X86 => Ok(()),
        InlineAsmArch::X86_64 => {
            Err("rbx is used internally by LLVM and cannot be used as an operand for inline asm")
        }
        _ => unreachable!(),
    }
}

fn esi_reserved(
    arch: InlineAsmArch,
    _target_features: &FxHashSet<Symbol>,
    _target: &Target,
    _is_clobber: bool,
) -> Result<(), &'static str> {
    match arch {
        InlineAsmArch::X86 => {
            Err("esi is used internally by LLVM and cannot be used as an operand for inline asm")
        }
        InlineAsmArch::X86_64 => Ok(()),
        _ => unreachable!(),
    }
}

def_regs! {
    X86 X86InlineAsmReg X86InlineAsmRegClass {
        ax: reg, reg_abcd = ["ax", "eax", "rax"],
        bx: reg, reg_abcd = ["bx", "ebx", "rbx"] % rbx_reserved,
        cx: reg, reg_abcd = ["cx", "ecx", "rcx"],
        dx: reg, reg_abcd = ["dx", "edx", "rdx"],
        si: reg = ["si", "esi", "rsi"] % esi_reserved,
        di: reg = ["di", "edi", "rdi"],
        r8: reg = ["r8", "r8w", "r8d"] % x86_64_only,
        r9: reg = ["r9", "r9w", "r9d"] % x86_64_only,
        r10: reg = ["r10", "r10w", "r10d"] % x86_64_only,
        r11: reg = ["r11", "r11w", "r11d"] % x86_64_only,
        r12: reg = ["r12", "r12w", "r12d"] % x86_64_only,
        r13: reg = ["r13", "r13w", "r13d"] % x86_64_only,
        r14: reg = ["r14", "r14w", "r14d"] % x86_64_only,
        r15: reg = ["r15", "r15w", "r15d"] % x86_64_only,
        al: reg_byte = ["al"],
        ah: reg_byte = ["ah"] % high_byte,
        bl: reg_byte = ["bl"],
        bh: reg_byte = ["bh"] % high_byte,
        cl: reg_byte = ["cl"],
        ch: reg_byte = ["ch"] % high_byte,
        dl: reg_byte = ["dl"],
        dh: reg_byte = ["dh"] % high_byte,
        sil: reg_byte = ["sil"] % x86_64_only,
        dil: reg_byte = ["dil"] % x86_64_only,
        r8b: reg_byte = ["r8b"] % x86_64_only,
        r9b: reg_byte = ["r9b"] % x86_64_only,
        r10b: reg_byte = ["r10b"] % x86_64_only,
        r11b: reg_byte = ["r11b"] % x86_64_only,
        r12b: reg_byte = ["r12b"] % x86_64_only,
        r13b: reg_byte = ["r13b"] % x86_64_only,
        r14b: reg_byte = ["r14b"] % x86_64_only,
        r15b: reg_byte = ["r15b"] % x86_64_only,
        xmm0: xmm_reg = ["xmm0"],
        xmm1: xmm_reg = ["xmm1"],
        xmm2: xmm_reg = ["xmm2"],
        xmm3: xmm_reg = ["xmm3"],
        xmm4: xmm_reg = ["xmm4"],
        xmm5: xmm_reg = ["xmm5"],
        xmm6: xmm_reg = ["xmm6"],
        xmm7: xmm_reg = ["xmm7"],
        xmm8: xmm_reg = ["xmm8"] % x86_64_only,
        xmm9: xmm_reg = ["xmm9"] % x86_64_only,
        xmm10: xmm_reg = ["xmm10"] % x86_64_only,
        xmm11: xmm_reg = ["xmm11"] % x86_64_only,
        xmm12: xmm_reg = ["xmm12"] % x86_64_only,
        xmm13: xmm_reg = ["xmm13"] % x86_64_only,
        xmm14: xmm_reg = ["xmm14"] % x86_64_only,
        xmm15: xmm_reg = ["xmm15"] % x86_64_only,
        ymm0: ymm_reg = ["ymm0"],
        ymm1: ymm_reg = ["ymm1"],
        ymm2: ymm_reg = ["ymm2"],
        ymm3: ymm_reg = ["ymm3"],
        ymm4: ymm_reg = ["ymm4"],
        ymm5: ymm_reg = ["ymm5"],
        ymm6: ymm_reg = ["ymm6"],
        ymm7: ymm_reg = ["ymm7"],
        ymm8: ymm_reg = ["ymm8"] % x86_64_only,
        ymm9: ymm_reg = ["ymm9"] % x86_64_only,
        ymm10: ymm_reg = ["ymm10"] % x86_64_only,
        ymm11: ymm_reg = ["ymm11"] % x86_64_only,
        ymm12: ymm_reg = ["ymm12"] % x86_64_only,
        ymm13: ymm_reg = ["ymm13"] % x86_64_only,
        ymm14: ymm_reg = ["ymm14"] % x86_64_only,
        ymm15: ymm_reg = ["ymm15"] % x86_64_only,
        zmm0: zmm_reg = ["zmm0"],
        zmm1: zmm_reg = ["zmm1"],
        zmm2: zmm_reg = ["zmm2"],
        zmm3: zmm_reg = ["zmm3"],
        zmm4: zmm_reg = ["zmm4"],
        zmm5: zmm_reg = ["zmm5"],
        zmm6: zmm_reg = ["zmm6"],
        zmm7: zmm_reg = ["zmm7"],
        zmm8: zmm_reg = ["zmm8"] % x86_64_only,
        zmm9: zmm_reg = ["zmm9"] % x86_64_only,
        zmm10: zmm_reg = ["zmm10"] % x86_64_only,
        zmm11: zmm_reg = ["zmm11"] % x86_64_only,
        zmm12: zmm_reg = ["zmm12"] % x86_64_only,
        zmm13: zmm_reg = ["zmm13"] % x86_64_only,
        zmm14: zmm_reg = ["zmm14"] % x86_64_only,
        zmm15: zmm_reg = ["zmm15"] % x86_64_only,
        zmm16: zmm_reg = ["zmm16", "xmm16", "ymm16"] % x86_64_only,
        zmm17: zmm_reg = ["zmm17", "xmm17", "ymm17"] % x86_64_only,
        zmm18: zmm_reg = ["zmm18", "xmm18", "ymm18"] % x86_64_only,
        zmm19: zmm_reg = ["zmm19", "xmm19", "ymm19"] % x86_64_only,
        zmm20: zmm_reg = ["zmm20", "xmm20", "ymm20"] % x86_64_only,
        zmm21: zmm_reg = ["zmm21", "xmm21", "ymm21"] % x86_64_only,
        zmm22: zmm_reg = ["zmm22", "xmm22", "ymm22"] % x86_64_only,
        zmm23: zmm_reg = ["zmm23", "xmm23", "ymm23"] % x86_64_only,
        zmm24: zmm_reg = ["zmm24", "xmm24", "ymm24"] % x86_64_only,
        zmm25: zmm_reg = ["zmm25", "xmm25", "ymm25"] % x86_64_only,
        zmm26: zmm_reg = ["zmm26", "xmm26", "ymm26"] % x86_64_only,
        zmm27: zmm_reg = ["zmm27", "xmm27", "ymm27"] % x86_64_only,
        zmm28: zmm_reg = ["zmm28", "xmm28", "ymm28"] % x86_64_only,
        zmm29: zmm_reg = ["zmm29", "xmm29", "ymm29"] % x86_64_only,
        zmm30: zmm_reg = ["zmm30", "xmm30", "ymm30"] % x86_64_only,
        zmm31: zmm_reg = ["zmm31", "xmm31", "ymm31"] % x86_64_only,
        k1: kreg = ["k1"],
        k2: kreg = ["k2"],
        k3: kreg = ["k3"],
        k4: kreg = ["k4"],
        k5: kreg = ["k5"],
        k6: kreg = ["k6"],
        k7: kreg = ["k7"],
        mm0: mmx_reg = ["mm0"],
        mm1: mmx_reg = ["mm1"],
        mm2: mmx_reg = ["mm2"],
        mm3: mmx_reg = ["mm3"],
        mm4: mmx_reg = ["mm4"],
        mm5: mmx_reg = ["mm5"],
        mm6: mmx_reg = ["mm6"],
        mm7: mmx_reg = ["mm7"],
        st0: x87_reg = ["st(0)", "st"],
        st1: x87_reg = ["st(1)"],
        st2: x87_reg = ["st(2)"],
        st3: x87_reg = ["st(3)"],
        st4: x87_reg = ["st(4)"],
        st5: x87_reg = ["st(5)"],
        st6: x87_reg = ["st(6)"],
        st7: x87_reg = ["st(7)"],
        #error = ["bp", "bpl", "ebp", "rbp"] =>
            "the frame pointer cannot be used as an operand for inline asm",
        #error = ["sp", "spl", "esp", "rsp"] =>
            "the stack pointer cannot be used as an operand for inline asm",
        #error = ["ip", "eip", "rip"] =>
            "the instruction pointer cannot be used as an operand for inline asm",
        #error = ["k0"] =>
            "the k0 AVX mask register cannot be used as an operand for inline asm",
    }
}

impl X86InlineAsmReg {
    pub fn emit(
        self,
        out: &mut dyn fmt::Write,
        arch: InlineAsmArch,
        modifier: Option<char>,
    ) -> fmt::Result {
        let reg_default_modifier = match arch {
            InlineAsmArch::X86 => 'e',
            InlineAsmArch::X86_64 => 'r',
            _ => unreachable!(),
        };
        if self as u32 <= Self::dx as u32 {
            let root = ['a', 'b', 'c', 'd'][self as usize - Self::ax as usize];
            match modifier.unwrap_or(reg_default_modifier) {
                'l' => write!(out, "{}l", root),
                'h' => write!(out, "{}h", root),
                'x' => write!(out, "{}x", root),
                'e' => write!(out, "e{}x", root),
                'r' => write!(out, "r{}x", root),
                _ => unreachable!(),
            }
        } else if self as u32 <= Self::di as u32 {
            let root = self.name();
            match modifier.unwrap_or(reg_default_modifier) {
                'l' => write!(out, "{}l", root),
                'x' => write!(out, "{}", root),
                'e' => write!(out, "e{}", root),
                'r' => write!(out, "r{}", root),
                _ => unreachable!(),
            }
        } else if self as u32 <= Self::r15 as u32 {
            let root = self.name();
            match modifier.unwrap_or(reg_default_modifier) {
                'l' => write!(out, "{}b", root),
                'x' => write!(out, "{}w", root),
                'e' => write!(out, "{}d", root),
                'r' => out.write_str(root),
                _ => unreachable!(),
            }
        } else if self as u32 <= Self::r15b as u32 {
            out.write_str(self.name())
        } else if self as u32 <= Self::xmm15 as u32 {
            let prefix = modifier.unwrap_or('x');
            let index = self as u32 - Self::xmm0 as u32;
            write!(out, "{}{}", prefix, index)
        } else if self as u32 <= Self::ymm15 as u32 {
            let prefix = modifier.unwrap_or('y');
            let index = self as u32 - Self::ymm0 as u32;
            write!(out, "{}{}", prefix, index)
        } else if self as u32 <= Self::zmm31 as u32 {
            let prefix = modifier.unwrap_or('z');
            let index = self as u32 - Self::zmm0 as u32;
            write!(out, "{}{}", prefix, index)
        } else {
            out.write_str(self.name())
        }
    }

    pub fn overlapping_regs(self, mut cb: impl FnMut(X86InlineAsmReg)) {
        macro_rules! reg_conflicts {
            (
                $(
                    $w:ident : $l:ident $h:ident
                ),*;
                $(
                    $w2:ident : $l2:ident
                ),*;
                $(
                    $x:ident : $y:ident : $z:ident
                ),*;
            ) => {
                match self {
                    $(
                        Self::$w => {
                            cb(Self::$w);
                            cb(Self::$l);
                            cb(Self::$h);
                        }
                        Self::$l => {
                            cb(Self::$w);
                            cb(Self::$l);
                        }
                        Self::$h => {
                            cb(Self::$w);
                            cb(Self::$h);
                        }
                    )*
                    $(
                        Self::$w2 | Self::$l2 => {
                            cb(Self::$w2);
                            cb(Self::$l2);
                        }
                    )*
                    $(
                        Self::$x | Self::$y | Self::$z => {
                            cb(Self::$x);
                            cb(Self::$y);
                            cb(Self::$z);
                        }
                    )*
                    r => cb(r),
                }
            };
        }

        // XMM*, YMM* and ZMM* are all different views of the same register.
        //
        // See section 15.5 of the combined Intel® 64 and IA-32 Architectures
        // Software Developer’s Manual for more details.
        //
        // We don't need to specify conflicts for [x,y,z]mm[16-31] since these
        // registers are only available with AVX-512, so we just specify them
        // as aliases directly.
        reg_conflicts! {
            ax : al ah,
            bx : bl bh,
            cx : cl ch,
            dx : dl dh;
            si : sil,
            di : dil,
            r8 : r8b,
            r9 : r9b,
            r10 : r10b,
            r11 : r11b,
            r12 : r12b,
            r13 : r13b,
            r14 : r14b,
            r15 : r15b;
            xmm0 : ymm0 : zmm0,
            xmm1 : ymm1 : zmm1,
            xmm2 : ymm2 : zmm2,
            xmm3 : ymm3 : zmm3,
            xmm4 : ymm4 : zmm4,
            xmm5 : ymm5 : zmm5,
            xmm6 : ymm6 : zmm6,
            xmm7 : ymm7 : zmm7,
            xmm8 : ymm8 : zmm8,
            xmm9 : ymm9 : zmm9,
            xmm10 : ymm10 : zmm10,
            xmm11 : ymm11 : zmm11,
            xmm12 : ymm12 : zmm12,
            xmm13 : ymm13 : zmm13,
            xmm14 : ymm14 : zmm14,
            xmm15 : ymm15 : zmm15;
        }
    }
}
Commit	Line	Data
f9f354fc	1	use super::{InlineAsmArch, InlineAsmType};
f035d41b	2	use crate::spec::Target;
5099ac24	3	use rustc_data_structures::stable_set::FxHashSet;
f9f354fc	4	use rustc_macros::HashStable_Generic;
5099ac24	5	use rustc_span::Symbol;
f9f354fc XL	6	use std::fmt;
	7
	8	def_reg_class! {
	9	X86 X86InlineAsmRegClass {
	10	reg,
	11	reg_abcd,
	12	reg_byte,
	13	xmm_reg,
	14	ymm_reg,
	15	zmm_reg,
	16	kreg,
136023e0 XL	17	mmx_reg,
136023e0 XL	18	x87_reg,
f9f354fc XL	19	}
	20	}
	21
	22	impl X86InlineAsmRegClass {
	23	pub fn valid_modifiers(self, arch: super::InlineAsmArch) -> &'static [char] {
	24	match self {
	25	Self::reg => {
	26	if arch == InlineAsmArch::X86_64 {
	27	&['l', 'x', 'e', 'r']
	28	} else {
	29	&['x', 'e']
	30	}
	31	}
	32	Self::reg_abcd => {
	33	if arch == InlineAsmArch::X86_64 {
	34	&['l', 'h', 'x', 'e', 'r']
	35	} else {
	36	&['l', 'h', 'x', 'e']
	37	}
	38	}
	39	Self::reg_byte => &[],
	40	Self::xmm_reg \| Self::ymm_reg \| Self::zmm_reg => &['x', 'y', 'z'],
	41	Self::kreg => &[],
136023e0	42	Self::mmx_reg \| Self::x87_reg => &[],
f9f354fc XL	43	}
	44	}
	45
	46	pub fn suggest_class(self, _arch: InlineAsmArch, ty: InlineAsmType) -> Option<Self> {
	47	match self {
	48	Self::reg \| Self::reg_abcd if ty.size().bits() == 8 => Some(Self::reg_byte),
	49	_ => None,
	50	}
	51	}
	52
	53	pub fn suggest_modifier(
	54	self,
	55	arch: InlineAsmArch,
	56	ty: InlineAsmType,
	57	) -> Option<(char, &'static str)> {
	58	match self {
	59	Self::reg => match ty.size().bits() {
	60	16 => Some(('x', "ax")),
	61	32 if arch == InlineAsmArch::X86_64 => Some(('e', "eax")),
	62	_ => None,
	63	},
	64	Self::reg_abcd => match ty.size().bits() {
	65	16 => Some(('x', "ax")),
	66	32 if arch == InlineAsmArch::X86_64 => Some(('e', "eax")),
	67	_ => None,
	68	},
	69	Self::reg_byte => None,
	70	Self::xmm_reg => None,
	71	Self::ymm_reg => match ty.size().bits() {
	72	256 => None,
	73	_ => Some(('x', "xmm0")),
	74	},
	75	Self::zmm_reg => match ty.size().bits() {
	76	512 => None,
	77	256 => Some(('y', "ymm0")),
	78	_ => Some(('x', "xmm0")),
	79	},
	80	Self::kreg => None,
136023e0	81	Self::mmx_reg \| Self::x87_reg => None,
f9f354fc XL	82	}
	83	}
	84
	85	pub fn default_modifier(self, arch: InlineAsmArch) -> Option<(char, &'static str)> {
	86	match self {
	87	Self::reg \| Self::reg_abcd => {
	88	if arch == InlineAsmArch::X86_64 {
	89	Some(('r', "rax"))
	90	} else {
	91	Some(('e', "eax"))
	92	}
	93	}
	94	Self::reg_byte => None,
	95	Self::xmm_reg => Some(('x', "xmm0")),
	96	Self::ymm_reg => Some(('y', "ymm0")),
	97	Self::zmm_reg => Some(('z', "zmm0")),
	98	Self::kreg => None,
136023e0	99	Self::mmx_reg \| Self::x87_reg => None,
f9f354fc XL	100	}
	101	}
	102
	103	pub fn supported_types(
	104	self,
	105	arch: InlineAsmArch,
5099ac24	106	) -> &'static [(InlineAsmType, Option<Symbol>)] {
f9f354fc XL	107	match self {
	108	Self::reg \| Self::reg_abcd => {
	109	if arch == InlineAsmArch::X86_64 {
	110	types! { _: I16, I32, I64, F32, F64; }
	111	} else {
	112	types! { _: I16, I32, F32; }
	113	}
	114	}
	115	Self::reg_byte => types! { _: I8; },
	116	Self::xmm_reg => types! {
5099ac24	117	sse: I32, I64, F32, F64,
f9f354fc XL	118	VecI8(16), VecI16(8), VecI32(4), VecI64(2), VecF32(4), VecF64(2);
	119	},
	120	Self::ymm_reg => types! {
5099ac24	121	avx: I32, I64, F32, F64,
f9f354fc XL	122	VecI8(16), VecI16(8), VecI32(4), VecI64(2), VecF32(4), VecF64(2),
	123	VecI8(32), VecI16(16), VecI32(8), VecI64(4), VecF32(8), VecF64(4);
	124	},
	125	Self::zmm_reg => types! {
5099ac24	126	avx512f: I32, I64, F32, F64,
f9f354fc XL	127	VecI8(16), VecI16(8), VecI32(4), VecI64(2), VecF32(4), VecF64(2),
	128	VecI8(32), VecI16(16), VecI32(8), VecI64(4), VecF32(8), VecF64(4),
	129	VecI8(64), VecI16(32), VecI32(16), VecI64(8), VecF32(16), VecF64(8);
	130	},
	131	Self::kreg => types! {
5099ac24 FG	132	avx512f: I8, I16;
5099ac24 FG	133	avx512bw: I32, I64;
f9f354fc	134	},
136023e0	135	Self::mmx_reg \| Self::x87_reg => &[],
f9f354fc XL	136	}
	137	}
	138	}
	139
	140	fn x86_64_only(
	141	arch: InlineAsmArch,
5099ac24	142	_target_features: &FxHashSet<Symbol>,
f035d41b	143	_target: &Target,
5099ac24	144	_is_clobber: bool,
f9f354fc XL	145	) -> Result<(), &'static str> {
	146	match arch {
	147	InlineAsmArch::X86 => Err("register is only available on x86_64"),
	148	InlineAsmArch::X86_64 => Ok(()),
	149	_ => unreachable!(),
	150	}
	151	}
	152
	153	fn high_byte(
	154	arch: InlineAsmArch,
5099ac24	155	_target_features: &FxHashSet<Symbol>,
f035d41b	156	_target: &Target,
5099ac24	157	_is_clobber: bool,
f9f354fc XL	158	) -> Result<(), &'static str> {
f9f354fc XL	159	match arch {
cdc7bbd5	160	InlineAsmArch::X86_64 => Err("high byte registers cannot be used as an operand on x86_64"),
f9f354fc XL	161	_ => Ok(()),
	162	}
	163	}
	164
17df50a5 XL	165	fn rbx_reserved(
17df50a5 XL	166	arch: InlineAsmArch,
5099ac24	167	_target_features: &FxHashSet<Symbol>,
17df50a5	168	_target: &Target,
5099ac24	169	_is_clobber: bool,
17df50a5 XL	170	) -> Result<(), &'static str> {
	171	match arch {
	172	InlineAsmArch::X86 => Ok(()),
	173	InlineAsmArch::X86_64 => {
	174	Err("rbx is used internally by LLVM and cannot be used as an operand for inline asm")
	175	}
	176	_ => unreachable!(),
	177	}
	178	}
	179
	180	fn esi_reserved(
	181	arch: InlineAsmArch,
5099ac24	182	_target_features: &FxHashSet<Symbol>,
17df50a5	183	_target: &Target,
5099ac24	184	_is_clobber: bool,
17df50a5 XL	185	) -> Result<(), &'static str> {
	186	match arch {
	187	InlineAsmArch::X86 => {
	188	Err("esi is used internally by LLVM and cannot be used as an operand for inline asm")
	189	}
	190	InlineAsmArch::X86_64 => Ok(()),
	191	_ => unreachable!(),
	192	}
	193	}
	194
f9f354fc XL	195	def_regs! {
	196	X86 X86InlineAsmReg X86InlineAsmRegClass {
	197	ax: reg, reg_abcd = ["ax", "eax", "rax"],
17df50a5	198	bx: reg, reg_abcd = ["bx", "ebx", "rbx"] % rbx_reserved,
f9f354fc XL	199	cx: reg, reg_abcd = ["cx", "ecx", "rcx"],
f9f354fc XL	200	dx: reg, reg_abcd = ["dx", "edx", "rdx"],
17df50a5	201	si: reg = ["si", "esi", "rsi"] % esi_reserved,
f9f354fc XL	202	di: reg = ["di", "edi", "rdi"],
	203	r8: reg = ["r8", "r8w", "r8d"] % x86_64_only,
	204	r9: reg = ["r9", "r9w", "r9d"] % x86_64_only,
	205	r10: reg = ["r10", "r10w", "r10d"] % x86_64_only,
	206	r11: reg = ["r11", "r11w", "r11d"] % x86_64_only,
	207	r12: reg = ["r12", "r12w", "r12d"] % x86_64_only,
	208	r13: reg = ["r13", "r13w", "r13d"] % x86_64_only,
	209	r14: reg = ["r14", "r14w", "r14d"] % x86_64_only,
	210	r15: reg = ["r15", "r15w", "r15d"] % x86_64_only,
	211	al: reg_byte = ["al"],
	212	ah: reg_byte = ["ah"] % high_byte,
	213	bl: reg_byte = ["bl"],
	214	bh: reg_byte = ["bh"] % high_byte,
	215	cl: reg_byte = ["cl"],
	216	ch: reg_byte = ["ch"] % high_byte,
	217	dl: reg_byte = ["dl"],
	218	dh: reg_byte = ["dh"] % high_byte,
	219	sil: reg_byte = ["sil"] % x86_64_only,
	220	dil: reg_byte = ["dil"] % x86_64_only,
	221	r8b: reg_byte = ["r8b"] % x86_64_only,
	222	r9b: reg_byte = ["r9b"] % x86_64_only,
	223	r10b: reg_byte = ["r10b"] % x86_64_only,
	224	r11b: reg_byte = ["r11b"] % x86_64_only,
	225	r12b: reg_byte = ["r12b"] % x86_64_only,
	226	r13b: reg_byte = ["r13b"] % x86_64_only,
	227	r14b: reg_byte = ["r14b"] % x86_64_only,
	228	r15b: reg_byte = ["r15b"] % x86_64_only,
	229	xmm0: xmm_reg = ["xmm0"],
	230	xmm1: xmm_reg = ["xmm1"],
	231	xmm2: xmm_reg = ["xmm2"],
	232	xmm3: xmm_reg = ["xmm3"],
	233	xmm4: xmm_reg = ["xmm4"],
	234	xmm5: xmm_reg = ["xmm5"],
	235	xmm6: xmm_reg = ["xmm6"],
	236	xmm7: xmm_reg = ["xmm7"],
	237	xmm8: xmm_reg = ["xmm8"] % x86_64_only,
	238	xmm9: xmm_reg = ["xmm9"] % x86_64_only,
	239	xmm10: xmm_reg = ["xmm10"] % x86_64_only,
	240	xmm11: xmm_reg = ["xmm11"] % x86_64_only,
	241	xmm12: xmm_reg = ["xmm12"] % x86_64_only,
	242	xmm13: xmm_reg = ["xmm13"] % x86_64_only,
	243	xmm14: xmm_reg = ["xmm14"] % x86_64_only,
	244	xmm15: xmm_reg = ["xmm15"] % x86_64_only,
	245	ymm0: ymm_reg = ["ymm0"],
	246	ymm1: ymm_reg = ["ymm1"],
	247	ymm2: ymm_reg = ["ymm2"],
	248	ymm3: ymm_reg = ["ymm3"],
	249	ymm4: ymm_reg = ["ymm4"],
	250	ymm5: ymm_reg = ["ymm5"],
	251	ymm6: ymm_reg = ["ymm6"],
	252	ymm7: ymm_reg = ["ymm7"],
	253	ymm8: ymm_reg = ["ymm8"] % x86_64_only,
	254	ymm9: ymm_reg = ["ymm9"] % x86_64_only,
	255	ymm10: ymm_reg = ["ymm10"] % x86_64_only,
	256	ymm11: ymm_reg = ["ymm11"] % x86_64_only,
	257	ymm12: ymm_reg = ["ymm12"] % x86_64_only,
	258	ymm13: ymm_reg = ["ymm13"] % x86_64_only,
	259	ymm14: ymm_reg = ["ymm14"] % x86_64_only,
	260	ymm15: ymm_reg = ["ymm15"] % x86_64_only,
	261	zmm0: zmm_reg = ["zmm0"],
	262	zmm1: zmm_reg = ["zmm1"],
	263	zmm2: zmm_reg = ["zmm2"],
	264	zmm3: zmm_reg = ["zmm3"],
	265	zmm4: zmm_reg = ["zmm4"],
266	zmm5: zmm_reg = ["zmm5"],
267	zmm6: zmm_reg = ["zmm6"],
268	zmm7: zmm_reg = ["zmm7"],
269	zmm8: zmm_reg = ["zmm8"] % x86_64_only,
270	zmm9: zmm_reg = ["zmm9"] % x86_64_only,
271	zmm10: zmm_reg = ["zmm10"] % x86_64_only,
272	zmm11: zmm_reg = ["zmm11"] % x86_64_only,
273	zmm12: zmm_reg = ["zmm12"] % x86_64_only,
274	zmm13: zmm_reg = ["zmm13"] % x86_64_only,
275	zmm14: zmm_reg = ["zmm14"] % x86_64_only,
276	zmm15: zmm_reg = ["zmm15"] % x86_64_only,
277	zmm16: zmm_reg = ["zmm16", "xmm16", "ymm16"] % x86_64_only,
278	zmm17: zmm_reg = ["zmm17", "xmm17", "ymm17"] % x86_64_only,
279	zmm18: zmm_reg = ["zmm18", "xmm18", "ymm18"] % x86_64_only,
280	zmm19: zmm_reg = ["zmm19", "xmm19", "ymm19"] % x86_64_only,
281	zmm20: zmm_reg = ["zmm20", "xmm20", "ymm20"] % x86_64_only,
282	zmm21: zmm_reg = ["zmm21", "xmm21", "ymm21"] % x86_64_only,
283	zmm22: zmm_reg = ["zmm22", "xmm22", "ymm22"] % x86_64_only,
284	zmm23: zmm_reg = ["zmm23", "xmm23", "ymm23"] % x86_64_only,
285	zmm24: zmm_reg = ["zmm24", "xmm24", "ymm24"] % x86_64_only,
286	zmm25: zmm_reg = ["zmm25", "xmm25", "ymm25"] % x86_64_only,
287	zmm26: zmm_reg = ["zmm26", "xmm26", "ymm26"] % x86_64_only,
288	zmm27: zmm_reg = ["zmm27", "xmm27", "ymm27"] % x86_64_only,
289	zmm28: zmm_reg = ["zmm28", "xmm28", "ymm28"] % x86_64_only,
290	zmm29: zmm_reg = ["zmm29", "xmm29", "ymm29"] % x86_64_only,
291	zmm30: zmm_reg = ["zmm30", "xmm30", "ymm30"] % x86_64_only,
292	zmm31: zmm_reg = ["zmm31", "xmm31", "ymm31"] % x86_64_only,
293	k1: kreg = ["k1"],
294	k2: kreg = ["k2"],
295	k3: kreg = ["k3"],
296	k4: kreg = ["k4"],
297	k5: kreg = ["k5"],
298	k6: kreg = ["k6"],
299	k7: kreg = ["k7"],
136023e0 XL	300	mm0: mmx_reg = ["mm0"],
	301	mm1: mmx_reg = ["mm1"],
	302	mm2: mmx_reg = ["mm2"],
	303	mm3: mmx_reg = ["mm3"],
	304	mm4: mmx_reg = ["mm4"],
	305	mm5: mmx_reg = ["mm5"],
	306	mm6: mmx_reg = ["mm6"],
	307	mm7: mmx_reg = ["mm7"],
	308	st0: x87_reg = ["st(0)", "st"],
	309	st1: x87_reg = ["st(1)"],
	310	st2: x87_reg = ["st(2)"],
	311	st3: x87_reg = ["st(3)"],
	312	st4: x87_reg = ["st(4)"],
	313	st5: x87_reg = ["st(5)"],
	314	st6: x87_reg = ["st(6)"],
	315	st7: x87_reg = ["st(7)"],
f9f354fc XL	316	#error = ["bp", "bpl", "ebp", "rbp"] =>
	317	"the frame pointer cannot be used as an operand for inline asm",
	318	#error = ["sp", "spl", "esp", "rsp"] =>
	319	"the stack pointer cannot be used as an operand for inline asm",
	320	#error = ["ip", "eip", "rip"] =>
	321	"the instruction pointer cannot be used as an operand for inline asm",
f9f354fc XL	322	#error = ["k0"] =>
	323	"the k0 AVX mask register cannot be used as an operand for inline asm",
	324	}
	325	}
	326
	327	impl X86InlineAsmReg {
	328	pub fn emit(
	329	self,
	330	out: &mut dyn fmt::Write,
	331	arch: InlineAsmArch,
	332	modifier: Option<char>,
	333	) -> fmt::Result {
	334	let reg_default_modifier = match arch {
	335	InlineAsmArch::X86 => 'e',
	336	InlineAsmArch::X86_64 => 'r',
	337	_ => unreachable!(),
	338	};
	339	if self as u32 <= Self::dx as u32 {
	340	let root = ['a', 'b', 'c', 'd'][self as usize - Self::ax as usize];
	341	match modifier.unwrap_or(reg_default_modifier) {
	342	'l' => write!(out, "{}l", root),
	343	'h' => write!(out, "{}h", root),
	344	'x' => write!(out, "{}x", root),
	345	'e' => write!(out, "e{}x", root),
	346	'r' => write!(out, "r{}x", root),
	347	_ => unreachable!(),
	348	}
	349	} else if self as u32 <= Self::di as u32 {
	350	let root = self.name();
	351	match modifier.unwrap_or(reg_default_modifier) {
	352	'l' => write!(out, "{}l", root),
	353	'x' => write!(out, "{}", root),
	354	'e' => write!(out, "e{}", root),
	355	'r' => write!(out, "r{}", root),
	356	_ => unreachable!(),
	357	}
	358	} else if self as u32 <= Self::r15 as u32 {
	359	let root = self.name();
	360	match modifier.unwrap_or(reg_default_modifier) {
	361	'l' => write!(out, "{}b", root),
	362	'x' => write!(out, "{}w", root),
	363	'e' => write!(out, "{}d", root),
	364	'r' => out.write_str(root),
	365	_ => unreachable!(),
	366	}
	367	} else if self as u32 <= Self::r15b as u32 {
	368	out.write_str(self.name())
	369	} else if self as u32 <= Self::xmm15 as u32 {
	370	let prefix = modifier.unwrap_or('x');
	371	let index = self as u32 - Self::xmm0 as u32;
	372	write!(out, "{}{}", prefix, index)
	373	} else if self as u32 <= Self::ymm15 as u32 {
	374	let prefix = modifier.unwrap_or('y');
	375	let index = self as u32 - Self::ymm0 as u32;
	376	write!(out, "{}{}", prefix, index)
	377	} else if self as u32 <= Self::zmm31 as u32 {
	378	let prefix = modifier.unwrap_or('z');
	379	let index = self as u32 - Self::zmm0 as u32;
	380	write!(out, "{}{}", prefix, index)
	381	} else {
	382	out.write_str(self.name())
	383	}
	384	}
	385
386	pub fn overlapping_regs(self, mut cb: impl FnMut(X86InlineAsmReg)) {
387	macro_rules! reg_conflicts {
388	(
389	$(
390	$w:ident : $l:ident $h:ident
391	),*;
392	$(
393	$w2:ident : $l2:ident
394	),*;
395	$(
396	$x:ident : $y:ident : $z:ident
397	),*;
398	) => {
399	match self {
400	$(
401	Self::$w => {
402	cb(Self::$w);
403	cb(Self::$l);
404	cb(Self::$h);
405	}
406	Self::$l => {
407	cb(Self::$w);
408	cb(Self::$l);
409	}
410	Self::$h => {
411	cb(Self::$w);
412	cb(Self::$h);
413	}
414	)*
415	$(
416	Self::$w2 \| Self::$l2 => {
417	cb(Self::$w2);
418	cb(Self::$l2);
419	}
420	)*
421	$(
422	Self::$x \| Self::$y \| Self::$z => {
423	cb(Self::$x);
424	cb(Self::$y);
425	cb(Self::$z);
426	}
427	)*
428	r => cb(r),
429	}
430	};
431	}
432
433	// XMM, YMM and ZMM* are all different views of the same register.
434	//
435	// See section 15.5 of the combined Intel® 64 and IA-32 Architectures
436	// Software Developer’s Manual for more details.
437	//
438	// We don't need to specify conflicts for [x,y,z]mm[16-31] since these
439	// registers are only available with AVX-512, so we just specify them
440	// as aliases directly.
441	reg_conflicts! {
442	ax : al ah,
443	bx : bl bh,
444	cx : cl ch,
445	dx : dl dh;
446	si : sil,
447	di : dil,
448	r8 : r8b,
449	r9 : r9b,
450	r10 : r10b,
451	r11 : r11b,
452	r12 : r12b,
453	r13 : r13b,
454	r14 : r14b,
455	r15 : r15b;
456	xmm0 : ymm0 : zmm0,
457	xmm1 : ymm1 : zmm1,
458	xmm2 : ymm2 : zmm2,
459	xmm3 : ymm3 : zmm3,
460	xmm4 : ymm4 : zmm4,
461	xmm5 : ymm5 : zmm5,
462	xmm6 : ymm6 : zmm6,
463	xmm7 : ymm7 : zmm7,
464	xmm8 : ymm8 : zmm8,
465	xmm9 : ymm9 : zmm9,
466	xmm10 : ymm10 : zmm10,
467	xmm11 : ymm11 : zmm11,
468	xmm12 : ymm12 : zmm12,
469	xmm13 : ymm13 : zmm13,
470	xmm14 : ymm14 : zmm14,
471	xmm15 : ymm15 : zmm15;
472	}
473	}
474	}