library/portable-simd/crates/core_simd/src/ops.rs

   1 use crate::simd::{LaneCount, Simd, SimdElement, SupportedLaneCount};
   2 use core::ops::{Add, Mul};
   3 use core::ops::{BitAnd, BitOr, BitXor};
   4 use core::ops::{Div, Rem, Sub};
   5 use core::ops::{Shl, Shr};
   6
   7 mod assign;
   8 mod deref;
   9 mod unary;
  10
  11 impl<I, T, const LANES: usize> core::ops::Index<I> for Simd<T, LANES>
  12 where
  13     T: SimdElement,
  14     LaneCount<LANES>: SupportedLaneCount,
  15     I: core::slice::SliceIndex<[T]>,
  16 {
  17     type Output = I::Output;
  18     fn index(&self, index: I) -> &Self::Output {
  19         &self.as_array()[index]
  20     }
  21 }
  22
  23 impl<I, T, const LANES: usize> core::ops::IndexMut<I> for Simd<T, LANES>
  24 where
  25     T: SimdElement,
  26     LaneCount<LANES>: SupportedLaneCount,
  27     I: core::slice::SliceIndex<[T]>,
  28 {
  29     fn index_mut(&mut self, index: I) -> &mut Self::Output {
  30         &mut self.as_mut_array()[index]
  31     }
  32 }
  33
  34 macro_rules! unsafe_base {
  35     ($lhs:ident, $rhs:ident, {$simd_call:ident}, $($_:tt)*) => {
  36         unsafe { $crate::simd::intrinsics::$simd_call($lhs, $rhs) }
  37     };
  38 }
  39
  40 /// SAFETY: This macro should not be used for anything except Shl or Shr, and passed the appropriate shift intrinsic.
  41 /// It handles performing a bitand in addition to calling the shift operator, so that the result
  42 /// is well-defined: LLVM can return a poison value if you shl, lshr, or ashr if rhs >= <Int>::BITS
  43 /// At worst, this will maybe add another instruction and cycle,
  44 /// at best, it may open up more optimization opportunities,
  45 /// or simply be elided entirely, especially for SIMD ISAs which default to this.
  46 ///
  47 // FIXME: Consider implementing this in cg_llvm instead?
  48 // cg_clif defaults to this, and scalar MIR shifts also default to wrapping
  49 macro_rules! wrap_bitshift {
  50     ($lhs:ident, $rhs:ident, {$simd_call:ident}, $int:ident) => {
  51         unsafe {
  52             $crate::simd::intrinsics::$simd_call(
  53                 $lhs,
  54                 $rhs.bitand(Simd::splat(<$int>::BITS as $int - 1)),
  55             )
  56         }
  57     };
  58 }
  59
  60 // Division by zero is poison, according to LLVM.
  61 // So is dividing the MIN value of a signed integer by -1,
  62 // since that would return MAX + 1.
  63 // FIXME: Rust allows <SInt>::MIN / -1,
  64 // so we should probably figure out how to make that safe.
  65 macro_rules! int_divrem_guard {
  66     (   $lhs:ident,
  67         $rhs:ident,
  68         {   const PANIC_ZERO: &'static str = $zero:literal;
  69             const PANIC_OVERFLOW: &'static str = $overflow:literal;
  70             $simd_call:ident
  71         },
  72         $int:ident ) => {
  73         if $rhs.lanes_eq(Simd::splat(0)).any() {
  74             panic!($zero);
  75         } else if <$int>::MIN != 0
  76             && ($lhs.lanes_eq(Simd::splat(<$int>::MIN))
  77                 // type inference can break here, so cut an SInt to size
  78                 & $rhs.lanes_eq(Simd::splat(-1i64 as _))).any()
  79         {
  80             panic!($overflow);
  81         } else {
  82             unsafe { $crate::simd::intrinsics::$simd_call($lhs, $rhs) }
  83         }
  84     };
  85 }
  86
  87 macro_rules! for_base_types {
  88     (   T = ($($scalar:ident),*);
  89         type Lhs = Simd<T, N>;
  90         type Rhs = Simd<T, N>;
  91         type Output = $out:ty;
  92
  93         impl $op:ident::$call:ident {
  94             $macro_impl:ident $inner:tt
  95         }) => {
  96             $(
  97                 impl<const N: usize> $op<Self> for Simd<$scalar, N>
  98                 where
  99                     $scalar: SimdElement,
 100                     LaneCount<N>: SupportedLaneCount,
 101                 {
 102                     type Output = $out;
 103
 104                     #[inline]
 105                     #[must_use = "operator returns a new vector without mutating the inputs"]
 106                     fn $call(self, rhs: Self) -> Self::Output {
 107                         $macro_impl!(self, rhs, $inner, $scalar)
 108                     }
 109                 })*
 110     }
 111 }
 112
 113 // A "TokenTree muncher": takes a set of scalar types `T = {};`
 114 // type parameters for the ops it implements, `Op::fn` names,
 115 // and a macro that expands into an expr, substituting in an intrinsic.
 116 // It passes that to for_base_types, which expands an impl for the types,
 117 // using the expanded expr in the function, and recurses with itself.
 118 //
 119 // tl;dr impls a set of ops::{Traits} for a set of types
 120 macro_rules! for_base_ops {
 121     (
 122         T = $types:tt;
 123         type Lhs = Simd<T, N>;
 124         type Rhs = Simd<T, N>;
 125         type Output = $out:ident;
 126         impl $op:ident::$call:ident
 127             $inner:tt
 128         $($rest:tt)*
 129     ) => {
 130         for_base_types! {
 131             T = $types;
 132             type Lhs = Simd<T, N>;
 133             type Rhs = Simd<T, N>;
 134             type Output = $out;
 135             impl $op::$call
 136                 $inner
 137         }
 138         for_base_ops! {
 139             T = $types;
 140             type Lhs = Simd<T, N>;
 141             type Rhs = Simd<T, N>;
 142             type Output = $out;
 143             $($rest)*
 144         }
 145     };
 146     ($($done:tt)*) => {
 147         // Done.
 148     }
 149 }
 150
 151 // Integers can always accept add, mul, sub, bitand, bitor, and bitxor.
 152 // For all of these operations, simd_* intrinsics apply wrapping logic.
 153 for_base_ops! {
 154     T = (i8, i16, i32, i64, isize, u8, u16, u32, u64, usize);
 155     type Lhs = Simd<T, N>;
 156     type Rhs = Simd<T, N>;
 157     type Output = Self;
 158
 159     impl Add::add {
 160         unsafe_base { simd_add }
 161     }
 162
 163     impl Mul::mul {
 164         unsafe_base { simd_mul }
 165     }
 166
 167     impl Sub::sub {
 168         unsafe_base { simd_sub }
 169     }
 170
 171     impl BitAnd::bitand {
 172         unsafe_base { simd_and }
 173     }
 174
 175     impl BitOr::bitor {
 176         unsafe_base { simd_or }
 177     }
 178
 179     impl BitXor::bitxor {
 180         unsafe_base { simd_xor }
 181     }
 182
 183     impl Div::div {
 184         int_divrem_guard {
 185             const PANIC_ZERO: &'static str = "attempt to divide by zero";
 186             const PANIC_OVERFLOW: &'static str = "attempt to divide with overflow";
 187             simd_div
 188         }
 189     }
 190
 191     impl Rem::rem {
 192         int_divrem_guard {
 193             const PANIC_ZERO: &'static str = "attempt to calculate the remainder with a divisor of zero";
 194             const PANIC_OVERFLOW: &'static str = "attempt to calculate the remainder with overflow";
 195             simd_rem
 196         }
 197     }
 198
 199     // The only question is how to handle shifts >= <Int>::BITS?
 200     // Our current solution uses wrapping logic.
 201     impl Shl::shl {
 202         wrap_bitshift { simd_shl }
 203     }
 204
 205     impl Shr::shr {
 206         wrap_bitshift {
 207             // This automatically monomorphizes to lshr or ashr, depending,
 208             // so it's fine to use it for both UInts and SInts.
 209             simd_shr
 210         }
 211     }
 212 }
 213
 214 // We don't need any special precautions here:
 215 // Floats always accept arithmetic ops, but may become NaN.
 216 for_base_ops! {
 217     T = (f32, f64);
 218     type Lhs = Simd<T, N>;
 219     type Rhs = Simd<T, N>;
 220     type Output = Self;
 221
 222     impl Add::add {
 223         unsafe_base { simd_add }
 224     }
 225
 226     impl Mul::mul {
 227         unsafe_base { simd_mul }
 228     }
 229
 230     impl Sub::sub {
 231         unsafe_base { simd_sub }
 232     }
 233
 234     impl Div::div {
 235         unsafe_base { simd_div }
 236     }
 237
 238     impl Rem::rem {
 239         unsafe_base { simd_rem }
 240     }
 241 }