ceph/src/erasure-code/isa/isa-l/erasure_code/gf_6vect_mad_avx.asm.s

   1 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
   2 ;  Copyright(c) 2011-2015 Intel Corporation All rights reserved.
   3 ;
   4 ;  Redistribution and use in source and binary forms, with or without
   5 ;  modification, are permitted provided that the following conditions
   6 ;  are met:
   7 ;    * Redistributions of source code must retain the above copyright
   8 ;      notice, this list of conditions and the following disclaimer.
   9 ;    * Redistributions in binary form must reproduce the above copyright
  10 ;      notice, this list of conditions and the following disclaimer in
  11 ;      the documentation and/or other materials provided with the
  12 ;      distribution.
  13 ;    * Neither the name of Intel Corporation nor the names of its
  14 ;      contributors may be used to endorse or promote products derived
  15 ;      from this software without specific prior written permission.
  16 ;
  17 ;  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18 ;  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19 ;  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20 ;  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21 ;  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22 ;  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23 ;  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24 ;  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25 ;  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26 ;  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27 ;  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  29
  30 ;;;
  31 ;;; gf_6vect_mad_avx(len, vec, vec_i, mul_array, src, dest);
  32 ;;;
  33
  34 %include "reg_sizes.asm"
  35
  36 %define PS 8
  37
  38 %ifidn __OUTPUT_FORMAT__, win64
  39  %define arg0  rcx
  40  %define arg0.w ecx
  41  %define arg1  rdx
  42  %define arg2  r8
  43  %define arg3  r9
  44  %define arg4  r12
  45  %define arg5  r15
  46  %define tmp   r11
  47  %define tmp2   r10
  48  %define tmp3   r13
  49  %define tmp4   r14
  50  %define tmp5   rdi
  51  %define return rax
  52  %define return.w eax
  53  %define stack_size 16*10 + 5*8
  54  %define arg(x)      [rsp + stack_size + PS + PS*x]
  55  %define func(x) proc_frame x
  56
  57 %macro FUNC_SAVE 0
  58         sub     rsp, stack_size
  59         movdqa  [rsp+16*0],xmm6
  60         movdqa  [rsp+16*1],xmm7
  61         movdqa  [rsp+16*2],xmm8
  62         movdqa  [rsp+16*3],xmm9
  63         movdqa  [rsp+16*4],xmm10
  64         movdqa  [rsp+16*5],xmm11
  65         movdqa  [rsp+16*6],xmm12
  66         movdqa  [rsp+16*7],xmm13
  67         movdqa  [rsp+16*8],xmm14
  68         movdqa  [rsp+16*9],xmm15
  69         save_reg        r12,  10*16 + 0*8
  70         save_reg        r13,  10*16 + 1*8
  71         save_reg        r14,  10*16 + 2*8
  72         save_reg        r15,  10*16 + 3*8
  73         save_reg        rdi,  10*16 + 4*8
  74         end_prolog
  75         mov     arg4, arg(4)
  76         mov     arg5, arg(5)
  77 %endmacro
  78
  79 %macro FUNC_RESTORE 0
  80         movdqa  xmm6, [rsp+16*0]
  81         movdqa  xmm7, [rsp+16*1]
  82         movdqa  xmm8, [rsp+16*2]
  83         movdqa  xmm9, [rsp+16*3]
  84         movdqa  xmm10, [rsp+16*4]
  85         movdqa  xmm11, [rsp+16*5]
  86         movdqa  xmm12, [rsp+16*6]
  87         movdqa  xmm13, [rsp+16*7]
  88         movdqa  xmm14, [rsp+16*8]
  89         movdqa  xmm15, [rsp+16*9]
  90         mov     r12,  [rsp + 10*16 + 0*8]
  91         mov     r13,  [rsp + 10*16 + 1*8]
  92         mov     r14,  [rsp + 10*16 + 2*8]
  93         mov     r15,  [rsp + 10*16 + 3*8]
  94         mov     rdi,  [rsp + 10*16 + 4*8]
  95         add     rsp, stack_size
  96 %endmacro
  97
  98 %elifidn __OUTPUT_FORMAT__, elf64
  99  %define arg0  rdi
 100  %define arg0.w edi
 101  %define arg1  rsi
 102  %define arg2  rdx
 103  %define arg3  rcx
 104  %define arg4  r8
 105  %define arg5  r9
 106  %define tmp   r11
 107  %define tmp2   r10
 108  %define tmp3   r12
 109  %define tmp4   r13
 110  %define tmp5   r14
 111  %define return rax
 112  %define return.w eax
 113
 114  %define func(x) x:
 115  %macro FUNC_SAVE 0
 116         push    r12
 117         push    r13
 118         push    r14
 119  %endmacro
 120  %macro FUNC_RESTORE 0
 121         pop     r14
 122         pop     r13
 123         pop     r12
 124  %endmacro
 125 %endif
 126
 127 ;;; gf_6vect_mad_avx(len, vec, vec_i, mul_array, src, dest)
 128 %define len   arg0
 129 %define len.w arg0.w
 130 %define vec    arg1
 131 %define vec_i    arg2
 132 %define mul_array arg3
 133 %define src   arg4
 134 %define dest1  arg5
 135 %define pos   return
 136 %define pos.w return.w
 137
 138 %define dest2 tmp4
 139 %define dest3 tmp2
 140 %define dest4 mul_array
 141 %define dest5 tmp5
 142 %define dest6 vec_i
 143
 144 %ifndef EC_ALIGNED_ADDR
 145 ;;; Use Un-aligned load/store
 146  %define XLDR vmovdqu
 147  %define XSTR vmovdqu
 148 %else
 149 ;;; Use Non-temporal load/stor
 150  %ifdef NO_NT_LDST
 151   %define XLDR vmovdqa
 152   %define XSTR vmovdqa
 153  %else
 154   %define XLDR vmovntdqa
 155   %define XSTR vmovntdq
 156  %endif
 157 %endif
 158
 159
 160 default rel
 161
 162 [bits 64]
 163 section .text
 164
 165 %define xmask0f  xmm15
 166 %define xgft4_lo  xmm14
 167 %define xgft4_hi  xmm13
 168 %define xgft5_lo  xmm12
 169 %define xgft5_hi  xmm11
 170 %define xgft6_lo  xmm10
 171 %define xgft6_hi  xmm9
 172
 173 %define x0         xmm0
 174 %define xtmpa      xmm1
 175 %define xtmph1     xmm2
 176 %define xtmpl1     xmm3
 177 %define xtmph2     xmm4
 178 %define xtmpl2     xmm5
 179 %define xtmph3     xmm6
 180 %define xtmpl3     xmm7
 181 %define xd1        xmm8
 182 %define xd2        xtmpl1
 183 %define xd3        xtmph1
 184
 185
 186 align 16
 187 global gf_6vect_mad_avx:function
 188 func(gf_6vect_mad_avx)
 189         FUNC_SAVE
 190         sub     len, 16
 191         jl      .return_fail
 192         xor     pos, pos
 193         vmovdqa xmask0f, [mask0f]       ;Load mask of lower nibble in each byte
 194         mov     tmp, vec
 195         sal     vec_i, 5                ;Multiply by 32
 196         lea     tmp3, [mul_array + vec_i]
 197         sal     tmp, 6                  ;Multiply by 64
 198
 199         sal     vec, 5                  ;Multiply by 32
 200         lea     vec_i, [tmp + vec]      ;vec_i = vec*96
 201         lea     mul_array, [tmp + vec_i]        ;mul_array = vec*160
 202
 203         vmovdqu xgft5_lo, [tmp3+2*tmp]          ;Load array Ex{00}, Ex{01}, ..., Ex{0f}
 204         vmovdqu xgft5_hi, [tmp3+2*tmp+16]       ;     "     Ex{00}, Ex{10}, ..., Ex{f0}
 205         vmovdqu xgft4_lo, [tmp3+vec_i]          ;Load array Dx{00}, Dx{01}, Dx{02}, ...
 206         vmovdqu xgft4_hi, [tmp3+vec_i+16]       ; " Dx{00}, Dx{10}, Dx{20}, ... , Dx{f0}
 207         vmovdqu xgft6_lo, [tmp3+mul_array]      ;Load array Fx{00}, Fx{01}, ..., Fx{0f}
 208         vmovdqu xgft6_hi, [tmp3+mul_array+16]   ;     "     Fx{00}, Fx{10}, ..., Fx{f0}
 209
 210         mov     dest2, [dest1+PS]
 211         mov     dest3, [dest1+2*PS]
 212         mov     dest4, [dest1+3*PS]  ; reuse mul_array
 213         mov     dest5, [dest1+4*PS]
 214         mov     dest6, [dest1+5*PS]  ; reuse vec_i
 215         mov     dest1, [dest1]
 216
 217 .loop16:
 218         XLDR    x0, [src+pos]           ;Get next source vector
 219
 220         vmovdqu xtmpl1, [tmp3]  ;Load array Ax{00}, Ax{01}, Ax{02}, ...
 221         vmovdqu xtmph1, [tmp3+16]       ; " Ax{00}, Ax{10}, Ax{20}, ... , Ax{f0}
 222         vmovdqu xtmpl2, [tmp3+vec]      ;Load array Bx{00}, Bx{01}, Bx{02}, ...
 223         vmovdqu xtmph2, [tmp3+vec+16]   ; " Bx{00}, Bx{10}, Bx{20}, ... , Bx{f0}
 224         vmovdqu xtmpl3, [tmp3+2*vec]    ;Load array Cx{00}, Cx{01}, Cx{02}, ...
 225         vmovdqu xtmph3, [tmp3+2*vec+16] ; " Cx{00}, Cx{10}, Cx{20}, ... , Cx{f0}
 226         XLDR    xd1, [dest1+pos]                ;Get next dest vector
 227
 228         vpand   xtmpa, x0, xmask0f      ;Mask low src nibble in bits 4-0
 229         vpsraw  x0, x0, 4               ;Shift to put high nibble into bits 4-0
 230         vpand   x0, x0, xmask0f         ;Mask high src nibble in bits 4-0
 231
 232
 233         ;dest1
 234         vpshufb xtmph1, x0              ;Lookup mul table of high nibble
 235         vpshufb xtmpl1, xtmpa           ;Lookup mul table of low nibble
 236         vpxor   xtmph1, xtmpl1          ;GF add high and low partials
 237         vpxor   xd1, xtmph1
 238
 239         XLDR    xd2, [dest2+pos]        ;reuse xtmpl1. Get next dest vector
 240         XLDR    xd3, [dest3+pos]        ;reuse xtmph1. Get next dest vector
 241
 242         ;dest2
 243         vpshufb xtmph2, x0              ;Lookup mul table of high nibble
 244         vpshufb xtmpl2, xtmpa           ;Lookup mul table of low nibble
 245         vpxor   xtmph2, xtmpl2          ;GF add high and low partials
 246         vpxor   xd2, xtmph2
 247
 248         ;dest3
 249         vpshufb xtmph3, x0              ;Lookup mul table of high nibble
 250         vpshufb xtmpl3, xtmpa           ;Lookup mul table of low nibble
 251         vpxor   xtmph3, xtmpl3          ;GF add high and low partials
 252         vpxor   xd3, xtmph3
 253
 254         XSTR    [dest1+pos], xd1        ;Store result into dest1
 255         XSTR    [dest2+pos], xd2        ;Store result into dest2
 256         XSTR    [dest3+pos], xd3        ;Store result into dest3
 257
 258         ;dest4
 259         XLDR    xd1, [dest4+pos]                ;Get next dest vector
 260         vpshufb xtmph1, xgft4_hi, x0            ;Lookup mul table of high nibble
 261         vpshufb xtmpl1, xgft4_lo, xtmpa         ;Lookup mul table of low nibble
 262         vpxor   xtmph1, xtmph1, xtmpl1          ;GF add high and low partials
 263         vpxor   xd1, xd1, xtmph1
 264
 265         XLDR    xd2, [dest5+pos]        ;reuse xtmpl1. Get next dest vector
 266         XLDR    xd3, [dest6+pos]        ;reuse xtmph1. Get next dest vector
 267
 268         ;dest5
 269         vpshufb xtmph2, xgft5_hi, x0            ;Lookup mul table of high nibble
 270         vpshufb xtmpl2, xgft5_lo, xtmpa         ;Lookup mul table of low nibble
 271         vpxor   xtmph2, xtmph2, xtmpl2          ;GF add high and low partials
 272         vpxor   xd2, xd2, xtmph2
 273
 274         ;dest6
 275         vpshufb xtmph3, xgft6_hi, x0            ;Lookup mul table of high nibble
 276         vpshufb xtmpl3, xgft6_lo, xtmpa         ;Lookup mul table of low nibble
 277         vpxor   xtmph3, xtmph3, xtmpl3          ;GF add high and low partials
 278         vpxor   xd3, xd3, xtmph3
 279
 280         XSTR    [dest4+pos], xd1        ;Store result into dest4
 281         XSTR    [dest5+pos], xd2        ;Store result into dest5
 282         XSTR    [dest6+pos], xd3        ;Store result into dest6
 283
 284         add     pos, 16                 ;Loop on 16 bytes at a time
 285         cmp     pos, len
 286         jle     .loop16
 287
 288         lea     tmp, [len + 16]
 289         cmp     pos, tmp
 290         je      .return_pass
 291
 292 .lessthan16:
 293         ;; Tail len
 294         ;; Do one more overlap pass
 295         ;; Overlapped offset length-16
 296         mov     tmp, len                ;Backup len as len=rdi
 297
 298         XLDR    x0, [src+tmp]           ;Get next source vector
 299         XLDR    xd1, [dest4+tmp]        ;Get next dest vector
 300         XLDR    xd2, [dest5+tmp]        ;reuse xtmpl1. Get next dest vector
 301         XLDR    xd3, [dest6+tmp]        ;reuse xtmph1. Get next dest vector
 302
 303         sub     len, pos
 304
 305         vmovdqa xtmph3, [constip16]     ;Load const of i + 16
 306         vpinsrb xtmpl3, len.w, 15
 307         vpshufb xtmpl3, xmask0f         ;Broadcast len to all bytes
 308         vpcmpgtb        xtmpl3, xtmpl3, xtmph3
 309
 310         vpand   xtmpa, x0, xmask0f      ;Mask low src nibble in bits 4-0
 311         vpsraw  x0, x0, 4               ;Shift to put high nibble into bits 4-0
 312         vpand   x0, x0, xmask0f         ;Mask high src nibble in bits 4-0
 313
 314         ;dest4
 315         vpshufb xgft4_hi, xgft4_hi, x0          ;Lookup mul table of high nibble
 316         vpshufb xgft4_lo, xgft4_lo, xtmpa               ;Lookup mul table of low nibble
 317         vpxor   xgft4_hi, xgft4_hi, xgft4_lo    ;GF add high and low partials
 318         vpand   xgft4_hi, xgft4_hi, xtmpl3
 319         vpxor   xd1, xd1, xgft4_hi
 320
 321         ;dest5
 322         vpshufb xgft5_hi, xgft5_hi, x0          ;Lookup mul table of high nibble
 323         vpshufb xgft5_lo, xgft5_lo, xtmpa               ;Lookup mul table of low nibble
 324         vpxor   xgft5_hi, xgft5_hi, xgft5_lo    ;GF add high and low partials
 325         vpand   xgft5_hi, xgft5_hi, xtmpl3
 326         vpxor   xd2, xd2, xgft5_hi
 327
 328         ;dest6
 329         vpshufb xgft6_hi, xgft6_hi, x0          ;Lookup mul table of high nibble
 330         vpshufb xgft6_lo, xgft6_lo, xtmpa               ;Lookup mul table of low nibble
 331         vpxor   xgft6_hi, xgft6_hi, xgft6_lo    ;GF add high and low partials
 332         vpand   xgft6_hi, xgft6_hi, xtmpl3
 333         vpxor   xd3, xd3, xgft6_hi
 334
 335         XSTR    [dest4+tmp], xd1        ;Store result into dest4
 336         XSTR    [dest5+tmp], xd2        ;Store result into dest5
 337         XSTR    [dest6+tmp], xd3        ;Store result into dest6
 338
 339         vmovdqu xgft4_lo, [tmp3]        ;Load array Ax{00}, Ax{01}, Ax{02}, ...
 340         vmovdqu xgft4_hi, [tmp3+16]     ; " Ax{00}, Ax{10}, Ax{20}, ... , Ax{f0}
 341         vmovdqu xgft5_lo, [tmp3+vec]    ;Load array Bx{00}, Bx{01}, Bx{02}, ...
 342         vmovdqu xgft5_hi, [tmp3+vec+16] ; " Bx{00}, Bx{10}, Bx{20}, ... , Bx{f0}
 343         vmovdqu xgft6_lo, [tmp3+2*vec]  ;Load array Cx{00}, Cx{01}, Cx{02}, ...
 344         vmovdqu xgft6_hi, [tmp3+2*vec+16]       ; " Cx{00}, Cx{10}, Cx{20}, ... , Cx{f0}
 345         XLDR    xd1, [dest1+tmp]        ;Get next dest vector
 346         XLDR    xd2, [dest2+tmp]        ;reuse xtmpl1. Get next dest vector
 347         XLDR    xd3, [dest3+tmp]        ;reuse xtmph1. Get next dest3 vector
 348
 349         ;dest1
 350         vpshufb xgft4_hi, xgft4_hi, x0          ;Lookup mul table of high nibble
 351         vpshufb xgft4_lo, xgft4_lo, xtmpa               ;Lookup mul table of low nibble
 352         vpxor   xgft4_hi, xgft4_hi, xgft4_lo            ;GF add high and low partials
 353         vpand   xgft4_hi, xgft4_hi, xtmpl3
 354         vpxor   xd1, xd1, xgft4_hi
 355
 356         ;dest2
 357         vpshufb xgft5_hi, xgft5_hi, x0          ;Lookup mul table of high nibble
 358         vpshufb xgft5_lo, xgft5_lo, xtmpa               ;Lookup mul table of low nibble
 359         vpxor   xgft5_hi, xgft5_hi, xgft5_lo    ;GF add high and low partials
 360         vpand   xgft5_hi, xgft5_hi, xtmpl3
 361         vpxor   xd2, xd2, xgft5_hi
 362
 363         ;dest3
 364         vpshufb xgft6_hi, xgft6_hi, x0          ;Lookup mul table of high nibble
 365         vpshufb xgft6_lo, xgft6_lo, xtmpa               ;Lookup mul table of low nibble
 366         vpxor   xgft6_hi, xgft6_hi, xgft6_lo    ;GF add high and low partials
 367         vpand   xgft6_hi, xgft6_hi, xtmpl3
 368         vpxor   xd3, xd3, xgft6_hi
 369
 370         XSTR    [dest1+tmp], xd1        ;Store result into dest1
 371         XSTR    [dest2+tmp], xd2        ;Store result into dest2
 372         XSTR    [dest3+tmp], xd3        ;Store result into dest3
 373
 374 .return_pass:
 375         FUNC_RESTORE
 376         mov     return, 0
 377         ret
 378
 379 .return_fail:
 380         FUNC_RESTORE
 381         mov     return, 1
 382         ret
 383
 384 endproc_frame
 385
 386 section .data
 387
 388 align 16
 389 mask0f: ddq 0x0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f
 390 constip16:
 391         ddq 0xf0f1f2f3f4f5f6f7f8f9fafbfcfdfeff
 392
 393 ;;;       func             core, ver, snum
 394 slversion gf_6vect_mad_avx, 02,  01,  0210