ceph/src/erasure-code/isa/isa-l/erasure_code/gf_4vect_mad_avx.asm.s

   1 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
   2 ;  Copyright(c) 2011-2015 Intel Corporation All rights reserved.
   3 ;
   4 ;  Redistribution and use in source and binary forms, with or without
   5 ;  modification, are permitted provided that the following conditions
   6 ;  are met:
   7 ;    * Redistributions of source code must retain the above copyright
   8 ;      notice, this list of conditions and the following disclaimer.
   9 ;    * Redistributions in binary form must reproduce the above copyright
  10 ;      notice, this list of conditions and the following disclaimer in
  11 ;      the documentation and/or other materials provided with the
  12 ;      distribution.
  13 ;    * Neither the name of Intel Corporation nor the names of its
  14 ;      contributors may be used to endorse or promote products derived
  15 ;      from this software without specific prior written permission.
  16 ;
  17 ;  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18 ;  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19 ;  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20 ;  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21 ;  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22 ;  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23 ;  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24 ;  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25 ;  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26 ;  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27 ;  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  29
  30 ;;;
  31 ;;; gf_4vect_mad_avx(len, vec, vec_i, mul_array, src, dest);
  32 ;;;
  33
  34 %include "reg_sizes.asm"
  35
  36 %define PS 8
  37
  38 %ifidn __OUTPUT_FORMAT__, win64
  39  %define arg0  rcx
  40  %define arg0.w ecx
  41  %define arg1  rdx
  42  %define arg2  r8
  43  %define arg3  r9
  44  %define arg4  r12
  45  %define arg5  r15
  46  %define tmp   r11
  47  %define tmp2   r10
  48  %define tmp3   r13
  49  %define return rax
  50  %define return.w eax
  51  %define stack_size 16*10 + 3*8
  52  %define arg(x)      [rsp + stack_size + PS + PS*x]
  53  %define func(x) proc_frame x
  54
  55 %macro FUNC_SAVE 0
  56         sub     rsp, stack_size
  57         movdqa  [rsp+16*0],xmm6
  58         movdqa  [rsp+16*1],xmm7
  59         movdqa  [rsp+16*2],xmm8
  60         movdqa  [rsp+16*3],xmm9
  61         movdqa  [rsp+16*4],xmm10
  62         movdqa  [rsp+16*5],xmm11
  63         movdqa  [rsp+16*6],xmm12
  64         movdqa  [rsp+16*7],xmm13
  65         movdqa  [rsp+16*8],xmm14
  66         movdqa  [rsp+16*9],xmm15
  67         save_reg        r12,  10*16 + 0*8
  68         save_reg        r13,  10*16 + 1*8
  69         save_reg        r15,  10*16 + 2*8
  70         end_prolog
  71         mov     arg4, arg(4)
  72         mov     arg5, arg(5)
  73 %endmacro
  74
  75 %macro FUNC_RESTORE 0
  76         movdqa  xmm6, [rsp+16*0]
  77         movdqa  xmm7, [rsp+16*1]
  78         movdqa  xmm8, [rsp+16*2]
  79         movdqa  xmm9, [rsp+16*3]
  80         movdqa  xmm10, [rsp+16*4]
  81         movdqa  xmm11, [rsp+16*5]
  82         movdqa  xmm12, [rsp+16*6]
  83         movdqa  xmm13, [rsp+16*7]
  84         movdqa  xmm14, [rsp+16*8]
  85         movdqa  xmm15, [rsp+16*9]
  86         mov     r12,  [rsp + 10*16 + 0*8]
  87         mov     r13,  [rsp + 10*16 + 1*8]
  88         mov     r15,  [rsp + 10*16 + 2*8]
  89         add     rsp, stack_size
  90 %endmacro
  91
  92 %elifidn __OUTPUT_FORMAT__, elf64
  93  %define arg0  rdi
  94  %define arg0.w edi
  95  %define arg1  rsi
  96  %define arg2  rdx
  97  %define arg3  rcx
  98  %define arg4  r8
  99  %define arg5  r9
 100  %define tmp   r11
 101  %define tmp2   r10
 102  %define tmp3   r12
 103  %define return rax
 104  %define return.w eax
 105
 106  %define func(x) x:
 107  %macro FUNC_SAVE 0
 108         push    r12
 109  %endmacro
 110  %macro FUNC_RESTORE 0
 111         pop     r12
 112  %endmacro
 113 %endif
 114
 115 ;;; gf_4vect_mad_avx(len, vec, vec_i, mul_array, src, dest)
 116 %define len   arg0
 117 %define len.w arg0.w
 118 %define vec    arg1
 119 %define vec_i    arg2
 120 %define mul_array arg3
 121 %define src   arg4
 122 %define dest1  arg5
 123 %define pos   return
 124 %define pos.w return.w
 125
 126 %define dest2 mul_array
 127 %define dest3 tmp2
 128 %define dest4 vec_i
 129
 130 %ifndef EC_ALIGNED_ADDR
 131 ;;; Use Un-aligned load/store
 132  %define XLDR vmovdqu
 133  %define XSTR vmovdqu
 134 %else
 135 ;;; Use Non-temporal load/stor
 136  %ifdef NO_NT_LDST
 137   %define XLDR vmovdqa
 138   %define XSTR vmovdqa
 139  %else
 140   %define XLDR vmovntdqa
 141   %define XSTR vmovntdq
 142  %endif
 143 %endif
 144
 145
 146 default rel
 147
 148 [bits 64]
 149 section .text
 150
 151 %define xmask0f  xmm15
 152 %define xgft3_hi xmm14
 153 %define xgft4_hi xmm13
 154 %define xgft4_lo xmm12
 155
 156 %define x0      xmm0
 157 %define xtmpa   xmm1
 158 %define xtmph1  xmm2
 159 %define xtmpl1  xmm3
 160 %define xtmph2  xmm4
 161 %define xtmpl2  xmm5
 162 %define xtmph3  xmm6
 163 %define xtmpl3  xmm7
 164 %define xtmph4  xmm8
 165 %define xtmpl4  xmm9
 166 %define xd1     xmm10
 167 %define xd2     xmm11
 168 %define xd3     xtmph1
 169 %define xd4     xtmpl1
 170
 171 align 16
 172 global gf_4vect_mad_avx:function
 173 func(gf_4vect_mad_avx)
 174         FUNC_SAVE
 175         sub     len, 16
 176         jl      .return_fail
 177         xor     pos, pos
 178         vmovdqa xmask0f, [mask0f]       ;Load mask of lower nibble in each byte
 179
 180         mov     tmp, vec
 181
 182         sal     vec_i, 5                ;Multiply by 32
 183         lea     tmp3, [mul_array + vec_i]
 184
 185         sal     tmp, 6                  ;Multiply by 64
 186         vmovdqu xgft3_hi, [tmp3+tmp+16] ; " Cx{00}, Cx{10}, Cx{20}, ... , Cx{f0}
 187         sal     vec, 5                  ;Multiply by 32
 188         add     tmp, vec
 189         vmovdqu xgft4_lo, [tmp3+tmp]    ;Load array Dx{00}, Dx{01}, Dx{02}, ...
 190         vmovdqu xgft4_hi, [tmp3+tmp+16] ; " Dx{00}, Dx{10}, Dx{20}, ... , Dx{f0}
 191
 192         mov     dest2, [dest1+PS]               ; reuse mul_array
 193         mov     dest3, [dest1+2*PS]
 194         mov     dest4, [dest1+3*PS]             ; reuse vec_i
 195         mov     dest1, [dest1]
 196
 197 .loop16:
 198         XLDR    x0, [src+pos]           ;Get next source vector
 199         vmovdqu xtmph1, [tmp3+16]       ; " Ax{00}, Ax{10}, Ax{20}, ... , Ax{f0}
 200         vmovdqu xtmpl1, [tmp3]          ;Load array Ax{00}, Ax{01}, Ax{02}, ...
 201         vmovdqu xtmph2, [tmp3+vec+16]   ; " Bx{00}, Bx{10}, Bx{20}, ... , Bx{f0}
 202         vmovdqu xtmpl2, [tmp3+vec]      ;Load array Bx{00}, Bx{01}, Bx{02}, ...
 203         vmovdqu xtmpl3, [tmp3+2*vec]    ;Load array Cx{00}, Cx{01}, Cx{02}, ...
 204
 205         XLDR    xd1, [dest1+pos]        ;Get next dest vector
 206         XLDR    xd2, [dest2+pos]        ;Get next dest vector
 207
 208         vpand   xtmpa, x0, xmask0f      ;Mask low src nibble in bits 4-0
 209         vpsraw  x0, x0, 4               ;Shift to put high nibble into bits 4-0
 210         vpand   x0, x0, xmask0f         ;Mask high src nibble in bits 4-0
 211
 212         ; dest1
 213         vpshufb xtmph1, xtmph1, x0              ;Lookup mul table of high nibble
 214         vpshufb xtmpl1, xtmpl1, xtmpa           ;Lookup mul table of low nibble
 215         vpxor   xtmph1, xtmph1, xtmpl1          ;GF add high and low partials
 216         vpxor   xd1, xd1, xtmph1
 217
 218         XLDR    xd3, [dest3+pos]        ;Reuse xtmph1, Get next dest vector
 219         XLDR    xd4, [dest4+pos]        ;Reuse xtmpl1, Get next dest vector
 220
 221         ; dest2
 222         vpshufb xtmph2, xtmph2, x0              ;Lookup mul table of high nibble
 223         vpshufb xtmpl2, xtmpl2, xtmpa           ;Lookup mul table of low nibble
 224         vpxor   xtmph2, xtmph2, xtmpl2          ;GF add high and low partials
 225         vpxor   xd2, xd2, xtmph2
 226
 227         ; dest3
 228         vpshufb xtmph3, xgft3_hi, x0            ;Lookup mul table of high nibble
 229         vpshufb xtmpl3, xtmpl3, xtmpa           ;Lookup mul table of low nibble
 230         vpxor   xtmph3, xtmph3, xtmpl3          ;GF add high and low partials
 231         vpxor   xd3, xd3, xtmph3
 232
 233         ; dest4
 234         vpshufb xtmph4, xgft4_hi, x0            ;Lookup mul table of high nibble
 235         vpshufb xtmpl4, xgft4_lo, xtmpa         ;Lookup mul table of low nibble
 236         vpxor   xtmph4, xtmph4, xtmpl4          ;GF add high and low partials
 237         vpxor   xd4, xd4, xtmph4
 238
 239         XSTR    [dest1+pos], xd1        ;Store result
 240         XSTR    [dest2+pos], xd2        ;Store result
 241         XSTR    [dest3+pos], xd3        ;Store result
 242         XSTR    [dest4+pos], xd4        ;Store result
 243
 244         add     pos, 16                 ;Loop on 16 bytes at a time
 245         cmp     pos, len
 246         jle     .loop16
 247
 248         lea     tmp, [len + 16]
 249         cmp     pos, tmp
 250         je      .return_pass
 251
 252 .lessthan16:
 253         ;; Tail len
 254         ;; Do one more overlap pass
 255
 256         mov     tmp, len        ;Overlapped offset length-16
 257
 258         XLDR    x0, [src+tmp]           ;Get next source vector
 259
 260         vmovdqu xtmph1, [tmp3+16]       ; " Ax{00}, Ax{10}, Ax{20}, ... , Ax{f0}
 261         vmovdqu xtmpl1, [tmp3]          ;Load array Ax{00}, Ax{01}, Ax{02}, ...
 262         vmovdqu xtmph2, [tmp3+vec+16]   ; " Bx{00}, Bx{10}, Bx{20}, ... , Bx{f0}
 263         vmovdqu xtmpl2, [tmp3+vec]      ;Load array Bx{00}, Bx{01}, Bx{02}, ...
 264         vmovdqu xtmpl3, [tmp3+2*vec]    ;Load array Cx{00}, Cx{01}, Cx{02}, ...
 265
 266         XLDR    xd1, [dest1+tmp]        ;Get next dest vector
 267         XLDR    xd2, [dest2+tmp]        ;Get next dest vector
 268         XLDR    xtmph4, [dest3+tmp]     ;Get next dest vector
 269
 270         sub     len, pos
 271
 272         vmovdqa xtmpl4, [constip16]     ;Load const of i + 16
 273         vpinsrb xtmph3, xtmph3, len.w, 15
 274         vpshufb xtmph3, xtmph3, xmask0f         ;Broadcast len to all bytes
 275         vpcmpgtb        xtmph3, xtmph3, xtmpl4
 276
 277         XLDR    xtmpl4, [dest4+tmp]     ;Get next dest vector
 278
 279         vpand   xtmpa, x0, xmask0f      ;Mask low src nibble in bits 4-0
 280         vpsraw  x0, x0, 4               ;Shift to put high nibble into bits 4-0
 281         vpand   x0, x0, xmask0f         ;Mask high src nibble in bits 4-0
 282
 283         ; dest1
 284         vpshufb xtmph1, xtmph1, x0      ;Lookup mul table of high nibble
 285         vpshufb xtmpl1, xtmpl1, xtmpa   ;Lookup mul table of low nibble
 286         vpxor   xtmph1, xtmph1, xtmpl1  ;GF add high and low partials
 287         vpand   xtmph1, xtmph1, xtmph3
 288         vpxor   xd1, xd1, xtmph1
 289
 290         ; dest2
 291         vpshufb xtmph2, xtmph2, x0      ;Lookup mul table of high nibble
 292         vpshufb xtmpl2, xtmpl2, xtmpa   ;Lookup mul table of low nibble
 293         vpxor   xtmph2, xtmph2, xtmpl2  ;GF add high and low partials
 294         vpand   xtmph2, xtmph2, xtmph3
 295         vpxor   xd2, xd2, xtmph2
 296
 297         ; dest3
 298         vpshufb xgft3_hi, xgft3_hi, x0          ;Lookup mul table of high nibble
 299         vpshufb xtmpl3, xtmpl3, xtmpa           ;Lookup mul table of low nibble
 300         vpxor   xgft3_hi, xgft3_hi, xtmpl3      ;GF add high and low partials
 301         vpand   xgft3_hi, xgft3_hi, xtmph3
 302         vpxor   xtmph4, xtmph4, xgft3_hi
 303
 304         ; dest4
 305         vpshufb xgft4_hi, xgft4_hi, x0          ;Lookup mul table of high nibble
 306         vpshufb xgft4_lo, xgft4_lo, xtmpa       ;Lookup mul table of low nibble
 307         vpxor   xgft4_hi, xgft4_hi, xgft4_lo    ;GF add high and low partials
 308         vpand   xgft4_hi, xgft4_hi, xtmph3
 309         vpxor   xtmpl4, xtmpl4, xgft4_hi
 310
 311         XSTR    [dest1+tmp], xd1        ;Store result
 312         XSTR    [dest2+tmp], xd2        ;Store result
 313         XSTR    [dest3+tmp], xtmph4     ;Store result
 314         XSTR    [dest4+tmp], xtmpl4     ;Store result
 315
 316 .return_pass:
 317         mov     return, 0
 318         FUNC_RESTORE
 319         ret
 320
 321 .return_fail:
 322         mov     return, 1
 323         FUNC_RESTORE
 324         ret
 325
 326 endproc_frame
 327
 328 section .data
 329
 330 align 16
 331 mask0f: ddq 0x0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f
 332 constip16:
 333         ddq 0xf0f1f2f3f4f5f6f7f8f9fafbfcfdfeff
 334
 335 ;;;       func             core, ver, snum
 336 slversion gf_4vect_mad_avx, 02,  01,  020a