ceph/src/erasure-code/isa/isa-l/erasure_code/gf_5vect_mad_sse.asm.s

   1 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
   2 ;  Copyright(c) 2011-2015 Intel Corporation All rights reserved.
   3 ;
   4 ;  Redistribution and use in source and binary forms, with or without
   5 ;  modification, are permitted provided that the following conditions
   6 ;  are met:
   7 ;    * Redistributions of source code must retain the above copyright
   8 ;      notice, this list of conditions and the following disclaimer.
   9 ;    * Redistributions in binary form must reproduce the above copyright
  10 ;      notice, this list of conditions and the following disclaimer in
  11 ;      the documentation and/or other materials provided with the
  12 ;      distribution.
  13 ;    * Neither the name of Intel Corporation nor the names of its
  14 ;      contributors may be used to endorse or promote products derived
  15 ;      from this software without specific prior written permission.
  16 ;
  17 ;  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18 ;  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19 ;  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20 ;  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21 ;  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22 ;  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23 ;  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24 ;  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25 ;  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26 ;  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27 ;  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  29
  30 ;;;
  31 ;;; gf_5vect_mad_sse(len, vec, vec_i, mul_array, src, dest);
  32 ;;;
  33
  34 %include "reg_sizes.asm"
  35
  36 %define PS 8
  37
  38 %ifidn __OUTPUT_FORMAT__, win64
  39  %define arg0  rcx
  40  %define arg0.w ecx
  41  %define arg1  rdx
  42  %define arg2  r8
  43  %define arg3  r9
  44  %define arg4  r12
  45  %define arg5  r15
  46  %define tmp   r11
  47  %define tmp2   r10
  48  %define tmp3   r13
  49  %define tmp4   r14
  50  %define return rax
  51  %define return.w eax
  52  %define stack_size 16*10 + 5*8
  53  %define arg(x)      [rsp + stack_size + PS + PS*x]
  54  %define func(x) proc_frame x
  55
  56 %macro FUNC_SAVE 0
  57         sub     rsp, stack_size
  58         movdqa  [rsp+16*0],xmm6
  59         movdqa  [rsp+16*1],xmm7
  60         movdqa  [rsp+16*2],xmm8
  61         movdqa  [rsp+16*3],xmm9
  62         movdqa  [rsp+16*4],xmm10
  63         movdqa  [rsp+16*5],xmm11
  64         movdqa  [rsp+16*6],xmm12
  65         movdqa  [rsp+16*7],xmm13
  66         movdqa  [rsp+16*8],xmm14
  67         movdqa  [rsp+16*9],xmm15
  68         save_reg        r12,  10*16 + 0*8
  69         save_reg        r13,  10*16 + 1*8
  70         save_reg        r14,  10*16 + 2*8
  71         save_reg        r15,  10*16 + 3*8
  72         end_prolog
  73         mov     arg4, arg(4)
  74         mov     arg5, arg(5)
  75 %endmacro
  76
  77 %macro FUNC_RESTORE 0
  78         movdqa  xmm6, [rsp+16*0]
  79         movdqa  xmm7, [rsp+16*1]
  80         movdqa  xmm8, [rsp+16*2]
  81         movdqa  xmm9, [rsp+16*3]
  82         movdqa  xmm10, [rsp+16*4]
  83         movdqa  xmm11, [rsp+16*5]
  84         movdqa  xmm12, [rsp+16*6]
  85         movdqa  xmm13, [rsp+16*7]
  86         movdqa  xmm14, [rsp+16*8]
  87         movdqa  xmm15, [rsp+16*9]
  88         mov     r12,  [rsp + 10*16 + 0*8]
  89         mov     r13,  [rsp + 10*16 + 1*8]
  90         mov     r14,  [rsp + 10*16 + 2*8]
  91         mov     r15,  [rsp + 10*16 + 3*8]
  92         add     rsp, stack_size
  93 %endmacro
  94
  95 %elifidn __OUTPUT_FORMAT__, elf64
  96  %define arg0  rdi
  97  %define arg0.w edi
  98  %define arg1  rsi
  99  %define arg2  rdx
 100  %define arg3  rcx
 101  %define arg4  r8
 102  %define arg5  r9
 103  %define tmp   r11
 104  %define tmp2   r10
 105  %define tmp3   r12
 106  %define tmp4   r13
 107  %define return rax
 108  %define return.w eax
 109
 110  %define func(x) x:
 111  %macro FUNC_SAVE 0
 112         push    r12
 113         push    r13
 114  %endmacro
 115  %macro FUNC_RESTORE 0
 116         pop     r13
 117         pop     r12
 118  %endmacro
 119 %endif
 120
 121 ;;; gf_5vect_mad_sse(len, vec, vec_i, mul_array, src, dest)
 122 %define len   arg0
 123 %define len.w arg0.w
 124 %define vec    arg1
 125 %define vec_i    arg2
 126 %define mul_array arg3
 127 %define src   arg4
 128 %define dest1  arg5
 129 %define pos   return
 130 %define pos.w return.w
 131
 132 %define dest2 tmp4
 133 %define dest3 mul_array
 134 %define dest4 tmp2
 135 %define dest5 vec_i
 136
 137 %ifndef EC_ALIGNED_ADDR
 138 ;;; Use Un-aligned load/store
 139  %define XLDR movdqu
 140  %define XSTR movdqu
 141 %else
 142 ;;; Use Non-temporal load/stor
 143  %ifdef NO_NT_LDST
 144   %define XLDR movdqa
 145   %define XSTR movdqa
 146  %else
 147   %define XLDR movntdqa
 148   %define XSTR movntdq
 149  %endif
 150 %endif
 151
 152 default rel
 153
 154 [bits 64]
 155 section .text
 156
 157 %define xmask0f  xmm15
 158 %define xgft5_hi xmm14
 159 %define xgft4_lo xmm13
 160 %define xgft4_hi xmm12
 161
 162 %define x0      xmm0
 163 %define xtmpa   xmm1
 164 %define xtmph1  xmm2
 165 %define xtmpl1  xmm3
 166 %define xtmph2  xmm4
 167 %define xtmpl2  xmm5
 168 %define xtmph3  xmm6
 169 %define xtmpl3  xmm7
 170 %define xtmph5  xmm8
 171 %define xtmpl5  xmm9
 172 %define xd1     xmm10
 173 %define xd2     xmm11
 174 %define xd3     xtmpl1
 175 %define xd4     xtmph1
 176 %define xd5     xtmpl2
 177
 178
 179 align 16
 180 global gf_5vect_mad_sse:function
 181 func(gf_5vect_mad_sse)
 182         FUNC_SAVE
 183         sub     len, 16
 184         jl      .return_fail
 185         xor     pos, pos
 186
 187         movdqa  xmask0f, [mask0f]       ;Load mask of lower nibble in each byte
 188         mov     tmp, vec
 189         sal     vec_i, 5                ;Multiply by 32
 190         lea     tmp3, [mul_array + vec_i]
 191         sal     tmp, 6                  ;Multiply by 64
 192         movdqu  xgft5_hi, [tmp3+2*tmp+16]       ;     "     Ex{00}, Ex{10}, ..., Ex{f0}
 193         sal     vec, 5                  ;Multiply by 32
 194         add     tmp, vec
 195         movdqu  xgft4_hi, [tmp3+tmp+16] ; " Dx{00}, Dx{10}, Dx{20}, ... , Dx{f0}
 196         movdqu  xgft4_lo, [tmp3+tmp]    ;Load array Dx{00}, Dx{01}, Dx{02}, ...
 197
 198         mov     dest3, [dest1+2*PS]     ; reuse mul_array
 199         mov     dest4, [dest1+3*PS]
 200         mov     dest5, [dest1+4*PS]     ; reuse vec_i
 201         mov     dest2, [dest1+PS]
 202         mov     dest1, [dest1]
 203
 204 .loop16:
 205         XLDR    x0, [src+pos]           ;Get next source vector
 206
 207         movdqu  xtmph1, [tmp3+16]       ; " Ax{00}, Ax{10}, Ax{20}, ... , Ax{f0}
 208         movdqu  xtmpl1, [tmp3]          ;Load array Ax{00}, Ax{01}, Ax{02}, ...
 209         movdqu  xtmph2, [tmp3+vec+16]   ; " Bx{00}, Bx{10}, Bx{20}, ... , Bx{f0}
 210         movdqu  xtmpl2, [tmp3+vec]      ;Load array Bx{00}, Bx{01}, Bx{02}, ...
 211         movdqu  xtmph3, [tmp3+2*vec+16] ; " Cx{00}, Cx{10}, Cx{20}, ... , Cx{f0}
 212         movdqu  xtmpl3, [tmp3+2*vec]    ;Load array Cx{00}, Cx{01}, Cx{02}, ...
 213         movdqu  xtmpl5, [tmp3+4*vec]    ;Load array Ex{00}, Ex{01}, ..., Ex{0f}
 214         movdqa  xtmph5, xgft5_hi                ;Reload const array registers
 215
 216         XLDR    xd1, [dest1+pos]        ;Get next dest vector
 217         XLDR    xd2, [dest2+pos]        ;Get next dest vector
 218
 219         movdqa  xtmpa, x0               ;Keep unshifted copy of src
 220         psraw   x0, 4                   ;Shift to put high nibble into bits 4-0
 221         pand    x0, xmask0f             ;Mask high src nibble in bits 4-0
 222         pand    xtmpa, xmask0f          ;Mask low src nibble in bits 4-0
 223
 224         ; dest1
 225         pshufb  xtmph1, x0              ;Lookup mul table of high nibble
 226         pshufb  xtmpl1, xtmpa           ;Lookup mul table of low nibble
 227         pxor    xtmph1, xtmpl1          ;GF add high and low partials
 228         pxor    xd1, xtmph1
 229
 230         XLDR    xd3, [dest3+pos]        ;Reuse xtmpl1, Get next dest vector
 231         XLDR    xd4, [dest4+pos]        ;Reuse xtmph1. Get next dest vector
 232
 233         ; dest2
 234         pshufb  xtmph2, x0              ;Lookup mul table of high nibble
 235         pshufb  xtmpl2, xtmpa           ;Lookup mul table of low nibble
 236         pxor    xtmph2, xtmpl2          ;GF add high and low partials
 237         pxor    xd2, xtmph2
 238
 239         XLDR    xd5, [dest5+pos]        ;Reuse xtmpl2. Get next dest vector
 240
 241         ; dest3
 242         pshufb  xtmph3, x0              ;Lookup mul table of high nibble
 243         pshufb  xtmpl3, xtmpa           ;Lookup mul table of low nibble
 244         pxor    xtmph3, xtmpl3          ;GF add high and low partials
 245         pxor    xd3, xtmph3
 246
 247         movdqa  xtmph2, xgft4_hi                ;Reload const array registers
 248         movdqa  xtmpl3, xgft4_lo                ;Reload const array registers
 249
 250         ; dest5
 251         pshufb  xtmph5, x0              ;Lookup mul table of high nibble
 252         pshufb  xtmpl5, xtmpa           ;Lookup mul table of low nibble
 253         pxor    xtmph5, xtmpl5          ;GF add high and low partials
 254         pxor    xd5, xtmph5
 255
 256         ; dest4
 257         pshufb  xtmph2, x0              ;Lookup mul table of high nibble
 258         pshufb  xtmpl3, xtmpa           ;Lookup mul table of low nibble
 259         pxor    xtmph2, xtmpl3          ;GF add high and low partials
 260         pxor    xd4, xtmph2
 261
 262         XSTR    [dest1+pos], xd1        ;Store result into dest1
 263         XSTR    [dest2+pos], xd2        ;Store result into dest2
 264         XSTR    [dest3+pos], xd3        ;Store result into dest3
 265         XSTR    [dest4+pos], xd4        ;Store result into dest4
 266         XSTR    [dest5+pos], xd5        ;Store result into dest5
 267
 268         add     pos, 16                 ;Loop on 16 bytes at a time
 269         cmp     pos, len
 270         jle     .loop16
 271
 272         lea     tmp, [len + 16]
 273         cmp     pos, tmp
 274         je      .return_pass
 275
 276 .lessthan16:
 277         ;; Tail len
 278         ;; Do one more overlap pass
 279         mov     tmp, len        ;Overlapped offset length-16
 280         XLDR    x0, [src+tmp]           ;Get next source vector
 281
 282         sub     len, pos
 283
 284         movdqa  xtmpl1, [constip16]     ;Load const of i + 16
 285         pinsrb  xtmph5, len.w, 15
 286         pshufb  xtmph5, xmask0f         ;Broadcast len to all bytes
 287         pcmpgtb xtmph5, xtmpl1
 288
 289         movdqu  xtmph1, [tmp3+16]       ; " Ax{00}, Ax{10}, Ax{20}, ... , Ax{f0}
 290         movdqu  xtmpl1, [tmp3]          ;Load array Ax{00}, Ax{01}, Ax{02}, ...
 291         movdqu  xtmph2, [tmp3+vec+16]   ; " Bx{00}, Bx{10}, Bx{20}, ... , Bx{f0}
 292         movdqu  xtmpl2, [tmp3+vec]      ;Load array Bx{00}, Bx{01}, Bx{02}, ...
 293         movdqu  xtmph3, [tmp3+2*vec+16] ; " Cx{00}, Cx{10}, Cx{20}, ... , Cx{f0}
 294         movdqu  xtmpl3, [tmp3+2*vec]    ;Load array Cx{00}, Cx{01}, Cx{02}, ...
 295         movdqu  xtmpl5, [tmp3+4*vec]    ;Load array Ex{00}, Ex{01}, ..., Ex{0f}
 296
 297         XLDR    xd1, [dest1+tmp]        ;Get next dest vector
 298         XLDR    xd2, [dest2+tmp]        ;Get next dest vector
 299
 300         movdqa  xtmpa, x0               ;Keep unshifted copy of src
 301         psraw   x0, 4                   ;Shift to put high nibble into bits 4-0
 302         pand    x0, xmask0f             ;Mask high src nibble in bits 4-0
 303         pand    xtmpa, xmask0f          ;Mask low src nibble in bits 4-0
 304
 305         ; dest1
 306         pshufb  xtmph1, x0              ;Lookup mul table of high nibble
 307         pshufb  xtmpl1, xtmpa           ;Lookup mul table of low nibble
 308         pxor    xtmph1, xtmpl1          ;GF add high and low partials
 309         pand    xtmph1, xtmph5
 310         pxor    xd1, xtmph1
 311
 312         XLDR    xd3, [dest3+tmp]        ;Reuse xtmpl1, Get next dest vector
 313         XLDR    xd4, [dest4+tmp]        ;Reuse xtmph1. Get next dest vector
 314
 315         ; dest2
 316         pshufb  xtmph2, x0              ;Lookup mul table of high nibble
 317         pshufb  xtmpl2, xtmpa           ;Lookup mul table of low nibble
 318         pxor    xtmph2, xtmpl2          ;GF add high and low partials
 319         pand    xtmph2, xtmph5
 320         pxor    xd2, xtmph2
 321
 322         XLDR    xd5, [dest5+tmp]        ;Reuse xtmpl2. Get next dest vector
 323
 324         ; dest3
 325         pshufb  xtmph3, x0              ;Lookup mul table of high nibble
 326         pshufb  xtmpl3, xtmpa           ;Lookup mul table of low nibble
 327         pxor    xtmph3, xtmpl3          ;GF add high and low partials
 328         pand    xtmph3, xtmph5
 329         pxor    xd3, xtmph3
 330
 331         ; dest4
 332         pshufb  xgft4_hi, x0            ;Lookup mul table of high nibble
 333         pshufb  xgft4_lo, xtmpa         ;Lookup mul table of low nibble
 334         pxor    xgft4_hi, xgft4_lo              ;GF add high and low partials
 335         pand    xgft4_hi, xtmph5
 336         pxor    xd4, xgft4_hi
 337
 338         ; dest5
 339         pshufb  xgft5_hi, x0            ;Lookup mul table of high nibble
 340         pshufb  xtmpl5, xtmpa           ;Lookup mul table of low nibble
 341         pxor    xgft5_hi, xtmpl5                ;GF add high and low partials
 342         pand    xgft5_hi, xtmph5
 343         pxor    xd5, xgft5_hi
 344
 345         XSTR    [dest1+tmp], xd1        ;Store result into dest1
 346         XSTR    [dest2+tmp], xd2        ;Store result into dest2
 347         XSTR    [dest3+tmp], xd3        ;Store result into dest3
 348         XSTR    [dest4+tmp], xd4        ;Store result into dest4
 349         XSTR    [dest5+tmp], xd5        ;Store result into dest5
 350
 351 .return_pass:
 352         FUNC_RESTORE
 353         mov     return, 0
 354         ret
 355
 356 .return_fail:
 357         FUNC_RESTORE
 358         mov     return, 1
 359         ret
 360
 361 endproc_frame
 362
 363 section .data
 364
 365 align 16
 366
 367 mask0f:
 368         ddq 0x0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f0f
 369 constip16:
 370         ddq 0xf0f1f2f3f4f5f6f7f8f9fafbfcfdfeff
 371
 372 ;;;       func             core, ver, snum
 373 slversion gf_5vect_mad_sse, 00,  01,  020c