compiler/rustc_target/src/abi/call/x86_64.rs

   1 // The classification code for the x86_64 ABI is taken from the clay language
   2 // https://github.com/jckarter/clay/blob/master/compiler/src/externals.cpp
   3
   4 use crate::abi::call::{ArgAbi, CastTarget, FnAbi, Reg, RegKind};
   5 use crate::abi::{self, Abi, HasDataLayout, LayoutOf, Size, TyAndLayout, TyAndLayoutMethods};
   6
   7 /// Classification of "eightbyte" components.
   8 // N.B., the order of the variants is from general to specific,
   9 // such that `unify(a, b)` is the "smaller" of `a` and `b`.
  10 #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)]
  11 enum Class {
  12     Int,
  13     Sse,
  14     SseUp,
  15 }
  16
  17 #[derive(Clone, Copy, Debug)]
  18 struct Memory;
  19
  20 // Currently supported vector size (AVX-512).
  21 const LARGEST_VECTOR_SIZE: usize = 512;
  22 const MAX_EIGHTBYTES: usize = LARGEST_VECTOR_SIZE / 64;
  23
  24 fn classify_arg<'a, Ty, C>(
  25     cx: &C,
  26     arg: &ArgAbi<'a, Ty>,
  27 ) -> Result<[Option<Class>; MAX_EIGHTBYTES], Memory>
  28 where
  29     Ty: TyAndLayoutMethods<'a, C> + Copy,
  30     C: LayoutOf<Ty = Ty, TyAndLayout = TyAndLayout<'a, Ty>> + HasDataLayout,
  31 {
  32     fn classify<'a, Ty, C>(
  33         cx: &C,
  34         layout: TyAndLayout<'a, Ty>,
  35         cls: &mut [Option<Class>],
  36         off: Size,
  37     ) -> Result<(), Memory>
  38     where
  39         Ty: TyAndLayoutMethods<'a, C> + Copy,
  40         C: LayoutOf<Ty = Ty, TyAndLayout = TyAndLayout<'a, Ty>> + HasDataLayout,
  41     {
  42         if !off.is_aligned(layout.align.abi) {
  43             if !layout.is_zst() {
  44                 return Err(Memory);
  45             }
  46             return Ok(());
  47         }
  48
  49         let mut c = match layout.abi {
  50             Abi::Uninhabited => return Ok(()),
  51
  52             Abi::Scalar(ref scalar) => match scalar.value {
  53                 abi::Int(..) | abi::Pointer => Class::Int,
  54                 abi::F32 | abi::F64 => Class::Sse,
  55             },
  56
  57             Abi::Vector { .. } => Class::Sse,
  58
  59             Abi::ScalarPair(..) | Abi::Aggregate { .. } => {
  60                 for i in 0..layout.fields.count() {
  61                     let field_off = off + layout.fields.offset(i);
  62                     classify(cx, layout.field(cx, i), cls, field_off)?;
  63                 }
  64
  65                 match &layout.variants {
  66                     abi::Variants::Single { .. } => {}
  67                     abi::Variants::Multiple { variants, .. } => {
  68                         // Treat enum variants like union members.
  69                         for variant_idx in variants.indices() {
  70                             classify(cx, layout.for_variant(cx, variant_idx), cls, off)?;
  71                         }
  72                     }
  73                 }
  74
  75                 return Ok(());
  76             }
  77         };
  78
  79         // Fill in `cls` for scalars (Int/Sse) and vectors (Sse).
  80         let first = (off.bytes() / 8) as usize;
  81         let last = ((off.bytes() + layout.size.bytes() - 1) / 8) as usize;
  82         for cls in &mut cls[first..=last] {
  83             *cls = Some(cls.map_or(c, |old| old.min(c)));
  84
  85             // Everything after the first Sse "eightbyte"
  86             // component is the upper half of a register.
  87             if c == Class::Sse {
  88                 c = Class::SseUp;
  89             }
  90         }
  91
  92         Ok(())
  93     }
  94
  95     let n = ((arg.layout.size.bytes() + 7) / 8) as usize;
  96     if n > MAX_EIGHTBYTES {
  97         return Err(Memory);
  98     }
  99
 100     let mut cls = [None; MAX_EIGHTBYTES];
 101     classify(cx, arg.layout, &mut cls, Size::ZERO)?;
 102     if n > 2 {
 103         if cls[0] != Some(Class::Sse) {
 104             return Err(Memory);
 105         }
 106         if cls[1..n].iter().any(|&c| c != Some(Class::SseUp)) {
 107             return Err(Memory);
 108         }
 109     } else {
 110         let mut i = 0;
 111         while i < n {
 112             if cls[i] == Some(Class::SseUp) {
 113                 cls[i] = Some(Class::Sse);
 114             } else if cls[i] == Some(Class::Sse) {
 115                 i += 1;
 116                 while i != n && cls[i] == Some(Class::SseUp) {
 117                     i += 1;
 118                 }
 119             } else {
 120                 i += 1;
 121             }
 122         }
 123     }
 124
 125     Ok(cls)
 126 }
 127
 128 fn reg_component(cls: &[Option<Class>], i: &mut usize, size: Size) -> Option<Reg> {
 129     if *i >= cls.len() {
 130         return None;
 131     }
 132
 133     match cls[*i] {
 134         None => None,
 135         Some(Class::Int) => {
 136             *i += 1;
 137             Some(if size.bytes() < 8 { Reg { kind: RegKind::Integer, size } } else { Reg::i64() })
 138         }
 139         Some(Class::Sse) => {
 140             let vec_len =
 141                 1 + cls[*i + 1..].iter().take_while(|&&c| c == Some(Class::SseUp)).count();
 142             *i += vec_len;
 143             Some(if vec_len == 1 {
 144                 match size.bytes() {
 145                     4 => Reg::f32(),
 146                     _ => Reg::f64(),
 147                 }
 148             } else {
 149                 Reg { kind: RegKind::Vector, size: Size::from_bytes(8) * (vec_len as u64) }
 150             })
 151         }
 152         Some(c) => unreachable!("reg_component: unhandled class {:?}", c),
 153     }
 154 }
 155
 156 fn cast_target(cls: &[Option<Class>], size: Size) -> CastTarget {
 157     let mut i = 0;
 158     let lo = reg_component(cls, &mut i, size).unwrap();
 159     let offset = Size::from_bytes(8) * (i as u64);
 160     let mut target = CastTarget::from(lo);
 161     if size > offset {
 162         if let Some(hi) = reg_component(cls, &mut i, size - offset) {
 163             target = CastTarget::pair(lo, hi);
 164         }
 165     }
 166     assert_eq!(reg_component(cls, &mut i, Size::ZERO), None);
 167     target
 168 }
 169
 170 const MAX_INT_REGS: usize = 6; // RDI, RSI, RDX, RCX, R8, R9
 171 const MAX_SSE_REGS: usize = 8; // XMM0-7
 172
 173 pub fn compute_abi_info<'a, Ty, C>(cx: &C, fn_abi: &mut FnAbi<'a, Ty>)
 174 where
 175     Ty: TyAndLayoutMethods<'a, C> + Copy,
 176     C: LayoutOf<Ty = Ty, TyAndLayout = TyAndLayout<'a, Ty>> + HasDataLayout,
 177 {
 178     let mut int_regs = MAX_INT_REGS;
 179     let mut sse_regs = MAX_SSE_REGS;
 180
 181     let mut x86_64_arg_or_ret = |arg: &mut ArgAbi<'a, Ty>, is_arg: bool| {
 182         let mut cls_or_mem = classify_arg(cx, arg);
 183
 184         if is_arg {
 185             if let Ok(cls) = cls_or_mem {
 186                 let mut needed_int = 0;
 187                 let mut needed_sse = 0;
 188                 for c in cls {
 189                     match c {
 190                         Some(Class::Int) => needed_int += 1,
 191                         Some(Class::Sse) => needed_sse += 1,
 192                         _ => {}
 193                     }
 194                 }
 195                 match (int_regs.checked_sub(needed_int), sse_regs.checked_sub(needed_sse)) {
 196                     (Some(left_int), Some(left_sse)) => {
 197                         int_regs = left_int;
 198                         sse_regs = left_sse;
 199                     }
 200                     _ => {
 201                         // Not enough registers for this argument, so it will be
 202                         // passed on the stack, but we only mark aggregates
 203                         // explicitly as indirect `byval` arguments, as LLVM will
 204                         // automatically put immediates on the stack itself.
 205                         if arg.layout.is_aggregate() {
 206                             cls_or_mem = Err(Memory);
 207                         }
 208                     }
 209                 }
 210             }
 211         }
 212
 213         match cls_or_mem {
 214             Err(Memory) => {
 215                 if is_arg {
 216                     arg.make_indirect_byval();
 217                 } else {
 218                     // `sret` parameter thus one less integer register available
 219                     arg.make_indirect();
 220                     // NOTE(eddyb) return is handled first, so no registers
 221                     // should've been used yet.
 222                     assert_eq!(int_regs, MAX_INT_REGS);
 223                     int_regs -= 1;
 224                 }
 225             }
 226             Ok(ref cls) => {
 227                 // split into sized chunks passed individually
 228                 if arg.layout.is_aggregate() {
 229                     let size = arg.layout.size;
 230                     arg.cast_to(cast_target(cls, size))
 231                 } else {
 232                     arg.extend_integer_width_to(32);
 233                 }
 234             }
 235         }
 236     };
 237
 238     if !fn_abi.ret.is_ignore() {
 239         x86_64_arg_or_ret(&mut fn_abi.ret, false);
 240     }
 241
 242     for arg in &mut fn_abi.args {
 243         if arg.is_ignore() {
 244             continue;
 245         }
 246         x86_64_arg_or_ret(arg, true);
 247     }
 248 }