ceph/src/isa-l/erasure_code/gf_vect_mad_avx2.asm

   1 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
   2 ;  Copyright(c) 2011-2015 Intel Corporation All rights reserved.
   3 ;
   4 ;  Redistribution and use in source and binary forms, with or without
   5 ;  modification, are permitted provided that the following conditions
   6 ;  are met:
   7 ;    * Redistributions of source code must retain the above copyright
   8 ;      notice, this list of conditions and the following disclaimer.
   9 ;    * Redistributions in binary form must reproduce the above copyright
  10 ;      notice, this list of conditions and the following disclaimer in
  11 ;      the documentation and/or other materials provided with the
  12 ;      distribution.
  13 ;    * Neither the name of Intel Corporation nor the names of its
  14 ;      contributors may be used to endorse or promote products derived
  15 ;      from this software without specific prior written permission.
  16 ;
  17 ;  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18 ;  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19 ;  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20 ;  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21 ;  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22 ;  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23 ;  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24 ;  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25 ;  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26 ;  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27 ;  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  29
  30 ;;;
  31 ;;; gf_vect_mad_avx2(len, vec, vec_i, mul_array, src, dest);
  32 ;;;
  33
  34 %include "reg_sizes.asm"
  35
  36 %ifidn __OUTPUT_FORMAT__, win64
  37  %define arg0   rcx
  38  %define arg0.w ecx
  39  %define arg1   rdx
  40  %define arg2   r8
  41  %define arg3   r9
  42  %define arg4   r12             ; must be saved and loaded
  43  %define arg5   r15
  44
  45  %define tmp    r11
  46  %define tmp.w  r11d
  47  %define tmp.b  r11b
  48  %define return rax
  49  %define return.w eax
  50  %define PS 8
  51  %define stack_size 16*3 + 3*8
  52  %define arg(x)      [rsp + stack_size + PS + PS*x]
  53  %define func(x) proc_frame x
  54
  55  %macro FUNC_SAVE 0
  56         sub     rsp, stack_size
  57         vmovdqa [rsp+16*0],xmm6
  58         vmovdqa [rsp+16*1],xmm7
  59         vmovdqa [rsp+16*2],xmm8
  60         save_reg        r12,  3*16 + 0*8
  61         save_reg        r15,  3*16 + 1*8
  62         end_prolog
  63         mov     arg4, arg(4)
  64         mov     arg5, arg(5)
  65  %endmacro
  66
  67  %macro FUNC_RESTORE 0
  68         vmovdqa xmm6, [rsp+16*0]
  69         vmovdqa xmm7, [rsp+16*1]
  70         vmovdqa xmm8, [rsp+16*2]
  71         mov     r12,  [rsp + 3*16 + 0*8]
  72         mov     r15,  [rsp + 3*16 + 1*8]
  73         add     rsp, stack_size
  74  %endmacro
  75
  76 %elifidn __OUTPUT_FORMAT__, elf64
  77  %define arg0  rdi
  78  %define arg0.w edi
  79  %define arg1  rsi
  80  %define arg2  rdx
  81  %define arg3  rcx
  82  %define arg4  r8
  83  %define arg5  r9
  84
  85  %define tmp      r11
  86  %define tmp.w    r11d
  87  %define tmp.b    r11b
  88  %define return rax
  89  %define return.w eax
  90
  91  %define func(x) x:
  92  %define FUNC_SAVE
  93  %define FUNC_RESTORE
  94 %endif
  95
  96
  97 ;;; gf_vect_mad_avx2(len, vec, vec_i, mul_array, src, dest)
  98 %define len   arg0
  99 %define len.w arg0.w
 100 %define vec   arg1
 101 %define vec_i    arg2
 102 %define mul_array arg3
 103 %define src   arg4
 104 %define dest  arg5
 105 %define pos   return
 106 %define pos.w return.w
 107
 108 %ifndef EC_ALIGNED_ADDR
 109 ;;; Use Un-aligned load/store
 110  %define XLDR vmovdqu
 111  %define XSTR vmovdqu
 112 %else
 113 ;;; Use Non-temporal load/stor
 114  %ifdef NO_NT_LDST
 115   %define XLDR vmovdqa
 116   %define XSTR vmovdqa
 117  %else
 118   %define XLDR vmovntdqa
 119   %define XSTR vmovntdq
 120  %endif
 121 %endif
 122
 123
 124 default rel
 125
 126 [bits 64]
 127 section .text
 128
 129 %define xmask0f  ymm8
 130 %define xmask0fx xmm8
 131 %define xgft_lo  ymm7
 132 %define xgft_hi  ymm6
 133
 134 %define x0     ymm0
 135 %define xtmpa  ymm1
 136 %define xtmph  ymm2
 137 %define xtmpl  ymm3
 138 %define xd     ymm4
 139 %define xtmpd  ymm5
 140
 141 align 16
 142 global gf_vect_mad_avx2:function
 143 func(gf_vect_mad_avx2)
 144         FUNC_SAVE
 145         sub     len, 32
 146         jl      .return_fail
 147         xor     pos, pos
 148         mov     tmp.b, 0x0f
 149         vpinsrb xmask0fx, xmask0fx, tmp.w, 0
 150         vpbroadcastb xmask0f, xmask0fx  ;Construct mask 0x0f0f0f...
 151
 152         sal     vec_i, 5                ;Multiply by 32
 153         vmovdqu xgft_lo, [vec_i+mul_array]      ;Load array Cx{00}, Cx{01}, Cx{02}, ...
 154                                                 ; " Cx{00}, Cx{10}, Cx{20}, ... , Cx{f0}
 155         vperm2i128 xgft_hi, xgft_lo, xgft_lo, 0x11 ; swapped to hi | hi
 156         vperm2i128 xgft_lo, xgft_lo, xgft_lo, 0x00 ; swapped to lo | lo
 157
 158         XLDR    xtmpd, [dest+len]       ;backup the last 32 bytes in dest
 159
 160 .loop32:
 161         XLDR    xd, [dest+pos]          ;Get next dest vector
 162 .loop32_overlap:
 163         XLDR    x0, [src+pos]           ;Get next source vector
 164
 165         vpand   xtmpa, x0, xmask0f      ;Mask low src nibble in bits 4-0
 166         vpsraw  x0, x0, 4               ;Shift to put high nibble into bits 4-0
 167         vpand   x0, x0, xmask0f         ;Mask high src nibble in bits 4-0
 168
 169         vpshufb xtmph, xgft_hi, x0      ;Lookup mul table of high nibble
 170         vpshufb xtmpl, xgft_lo, xtmpa   ;Lookup mul table of low nibble
 171         vpxor   xtmph, xtmph, xtmpl ;GF add high and low partials
 172         vpxor   xd, xd, xtmph           ;xd += partial
 173
 174         XSTR    [dest+pos], xd
 175         add     pos, 32                 ;Loop on 32 bytes at a time
 176         cmp     pos, len
 177         jle     .loop32
 178
 179         lea     tmp, [len + 32]
 180         cmp     pos, tmp
 181         je      .return_pass
 182
 183         ;; Tail len
 184         mov     pos, len        ;Overlapped offset length-32
 185         vmovdqa xd, xtmpd       ;Restore xd
 186         jmp     .loop32_overlap ;Do one more overlap pass
 187
 188 .return_pass:
 189         mov     return, 0
 190         FUNC_RESTORE
 191         ret
 192
 193 .return_fail:
 194         mov     return, 1
 195         FUNC_RESTORE
 196         ret
 197
 198 endproc_frame
 199
 200 section .data
 201
 202 ;;;       func             core, ver, snum
 203 slversion gf_vect_mad_avx2, 04,  01,  0202