ceph/src/isa-l/erasure_code/gf_4vect_mad_avx512.asm

   1 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
   2 ;  Copyright(c) 2011-2015 Intel Corporation All rights reserved.
   3 ;
   4 ;  Redistribution and use in source and binary forms, with or without
   5 ;  modification, are permitted provided that the following conditions
   6 ;  are met:
   7 ;    * Redistributions of source code must retain the above copyright
   8 ;      notice, this list of conditions and the following disclaimer.
   9 ;    * Redistributions in binary form must reproduce the above copyright
  10 ;      notice, this list of conditions and the following disclaimer in
  11 ;      the documentation and/or other materials provided with the
  12 ;      distribution.
  13 ;    * Neither the name of Intel Corporation nor the names of its
  14 ;      contributors may be used to endorse or promote products derived
  15 ;      from this software without specific prior written permission.
  16 ;
  17 ;  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18 ;  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19 ;  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20 ;  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21 ;  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22 ;  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23 ;  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24 ;  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25 ;  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26 ;  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27 ;  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  29
  30 ;;;
  31 ;;; gf_4vect_mad_avx512(len, vec, vec_i, mul_array, src, dest);
  32 ;;;
  33
  34 %include "reg_sizes.asm"
  35
  36 %ifdef HAVE_AS_KNOWS_AVX512
  37
  38 %ifidn __OUTPUT_FORMAT__, elf64
  39  %define arg0   rdi
  40  %define arg1   rsi
  41  %define arg2   rdx
  42  %define arg3   rcx
  43  %define arg4   r8
  44  %define arg5   r9
  45  %define tmp    r11
  46  %define return rax
  47  %define func(x) x:
  48  %define FUNC_SAVE
  49  %define FUNC_RESTORE
  50 %endif
  51
  52 %ifidn __OUTPUT_FORMAT__, win64
  53  %define arg0   rcx
  54  %define arg1   rdx
  55  %define arg2   r8
  56  %define arg3   r9
  57  %define arg4   r12
  58  %define arg5   r15
  59  %define tmp    r11
  60  %define return rax
  61  %define stack_size 16*10 + 3*8
  62  %define arg(x)      [rsp + stack_size + PS + PS*x]
  63  %define func(x) proc_frame x
  64
  65 %macro FUNC_SAVE 0
  66         sub     rsp, stack_size
  67         movdqa  [rsp+16*0],xmm6
  68         movdqa  [rsp+16*1],xmm7
  69         movdqa  [rsp+16*2],xmm8
  70         movdqa  [rsp+16*3],xmm9
  71         movdqa  [rsp+16*4],xmm10
  72         movdqa  [rsp+16*5],xmm11
  73         movdqa  [rsp+16*6],xmm12
  74         movdqa  [rsp+16*7],xmm13
  75         movdqa  [rsp+16*8],xmm14
  76         movdqa  [rsp+16*9],xmm15
  77         save_reg        r12,  10*16 + 0*8
  78         save_reg        r15,  10*16 + 1*8
  79         end_prolog
  80         mov     arg4, arg(4)
  81         mov     arg5, arg(5)
  82 %endmacro
  83
  84 %macro FUNC_RESTORE 0
  85         movdqa  xmm6, [rsp+16*0]
  86         movdqa  xmm7, [rsp+16*1]
  87         movdqa  xmm8, [rsp+16*2]
  88         movdqa  xmm9, [rsp+16*3]
  89         movdqa  xmm10, [rsp+16*4]
  90         movdqa  xmm11, [rsp+16*5]
  91         movdqa  xmm12, [rsp+16*6]
  92         movdqa  xmm13, [rsp+16*7]
  93         movdqa  xmm14, [rsp+16*8]
  94         movdqa  xmm15, [rsp+16*9]
  95         mov     r12,  [rsp + 10*16 + 0*8]
  96         mov     r15,  [rsp + 10*16 + 1*8]
  97         add     rsp, stack_size
  98 %endmacro
  99 %endif
 100
 101 %define PS    8
 102 %define len   arg0
 103 %define vec   arg1
 104 %define vec_i arg2
 105 %define mul_array arg3
 106 %define src   arg4
 107 %define dest1 arg5
 108 %define pos   return
 109 %define dest2 mul_array
 110 %define dest3 vec
 111 %define dest4 vec_i
 112
 113 %ifndef EC_ALIGNED_ADDR
 114 ;;; Use Un-aligned load/store
 115  %define XLDR vmovdqu8
 116  %define XSTR vmovdqu8
 117 %else
 118 ;;; Use Non-temporal load/stor
 119  %ifdef NO_NT_LDST
 120   %define XLDR vmovdqa
 121   %define XSTR vmovdqa
 122  %else
 123   %define XLDR vmovntdqa
 124   %define XSTR vmovntdq
 125  %endif
 126 %endif
 127
 128 default rel
 129 [bits 64]
 130 section .text
 131
 132 %define x0        zmm0
 133 %define xtmpa     zmm1
 134 %define xtmpl1    zmm2
 135 %define xtmph1    zmm3
 136 %define xtmph2    zmm4
 137 %define xtmph3    zmm5
 138 %define xtmph4    zmm6
 139 %define xgft1_hi  zmm7
 140 %define xgft1_lo  zmm8
 141 %define xgft1_loy ymm8
 142 %define xgft2_hi  zmm9
 143 %define xgft2_lo  zmm10
 144 %define xgft2_loy ymm10
 145 %define xgft3_hi  zmm11
 146 %define xgft3_lo  zmm12
 147 %define xgft3_loy ymm12
 148 %define xgft4_hi  zmm13
 149 %define xgft4_lo  zmm14
 150 %define xgft4_loy ymm14
 151 %define xd1       zmm15
 152 %define xd2       zmm16
 153 %define xd3       zmm17
 154 %define xd4       zmm18
 155 %define xmask0f   zmm19
 156 %define xtmpl2    zmm20
 157 %define xtmpl3    zmm21
 158 %define xtmpl4    zmm22
 159 %define xtmpl5    zmm23
 160
 161 align 16
 162 global gf_4vect_mad_avx512:function
 163 func(gf_4vect_mad_avx512)
 164         FUNC_SAVE
 165         sub     len, 64
 166         jl      .return_fail
 167         xor     pos, pos
 168         mov     tmp, 0x0f
 169         vpbroadcastb xmask0f, tmp       ;Construct mask 0x0f0f0f...
 170         sal     vec_i, 5                ;Multiply by 32
 171         sal     vec, 5                  ;Multiply by 32
 172         lea     tmp, [mul_array + vec_i]
 173         vmovdqu xgft1_loy, [tmp]        ;Load array Ax{00}..{0f}, Ax{00}..{f0}
 174         vmovdqu xgft2_loy, [tmp+vec]    ;Load array Bx{00}..{0f}, Bx{00}..{f0}
 175         vmovdqu xgft3_loy, [tmp+2*vec]  ;Load array Cx{00}..{0f}, Cx{00}..{f0}
 176         add     tmp, vec
 177         vmovdqu xgft4_loy, [tmp+2*vec]  ;Load array Dx{00}..{0f}, Dx{00}..{f0}
 178         vshufi64x2 xgft1_hi, xgft1_lo, xgft1_lo, 0x55
 179         vshufi64x2 xgft1_lo, xgft1_lo, xgft1_lo, 0x00
 180         vshufi64x2 xgft2_hi, xgft2_lo, xgft2_lo, 0x55
 181         vshufi64x2 xgft2_lo, xgft2_lo, xgft2_lo, 0x00
 182         vshufi64x2 xgft3_hi, xgft3_lo, xgft3_lo, 0x55
 183         vshufi64x2 xgft3_lo, xgft3_lo, xgft3_lo, 0x00
 184         vshufi64x2 xgft4_hi, xgft4_lo, xgft4_lo, 0x55
 185         vshufi64x2 xgft4_lo, xgft4_lo, xgft4_lo, 0x00
 186         mov     dest2, [dest1+PS]               ; reuse mul_array
 187         mov     dest3, [dest1+2*PS]             ; reuse vec
 188         mov     dest4, [dest1+3*PS]             ; reuse vec_i
 189         mov     dest1, [dest1]
 190         mov     tmp, -1
 191         kmovq   k1, tmp
 192
 193 .loop64:
 194         XLDR    x0, [src+pos]           ;Get next source vector
 195         XLDR    xd1, [dest1+pos]        ;Get next dest vector
 196         XLDR    xd2, [dest2+pos]        ;Get next dest vector
 197         XLDR    xd3, [dest3+pos]        ;Get next dest vector
 198         XLDR    xd4, [dest4+pos]        ;reuse xtmpl1. Get next dest vector
 199
 200         vpandq  xtmpa, x0, xmask0f      ;Mask low src nibble in bits 4-0
 201         vpsraw  x0, x0, 4               ;Shift to put high nibble into bits 4-0
 202         vpandq  x0, x0, xmask0f         ;Mask high src nibble in bits 4-0
 203
 204         ; dest1
 205         vpshufb xtmph1 {k1}{z}, xgft1_hi, x0    ;Lookup mul table of high nibble
 206         vpshufb xtmpl1 {k1}{z}, xgft1_lo, xtmpa ;Lookup mul table of low nibble
 207         vpxorq  xtmph1, xtmph1, xtmpl1          ;GF add high and low partials
 208         vpxorq  xd1, xd1, xtmph1                ;xd1 += partial
 209
 210         ; dest2
 211         vpshufb xtmph2 {k1}{z}, xgft2_hi, x0    ;Lookup mul table of high nibble
 212         vpshufb xtmpl2 {k1}{z}, xgft2_lo, xtmpa ;Lookup mul table of low nibble
 213         vpxorq  xtmph2, xtmph2, xtmpl2          ;GF add high and low partials
 214         vpxorq  xd2, xd2, xtmph2                ;xd2 += partial
 215
 216         ; dest3
 217         vpshufb xtmph3 {k1}{z}, xgft3_hi, x0    ;Lookup mul table of high nibble
 218         vpshufb xtmpl3 {k1}{z}, xgft3_lo, xtmpa ;Lookup mul table of low nibble
 219         vpxorq  xtmph3, xtmph3, xtmpl3          ;GF add high and low partials
 220         vpxorq  xd3, xd3, xtmph3                ;xd2 += partial
 221
 222         ; dest4
 223         vpshufb xtmph4 {k1}{z}, xgft4_hi, x0    ;Lookup mul table of high nibble
 224         vpshufb xtmpl4 {k1}{z}, xgft4_lo, xtmpa ;Lookup mul table of low nibble
 225         vpxorq  xtmph4, xtmph4, xtmpl4          ;GF add high and low partials
 226         vpxorq  xd4, xd4, xtmph4                ;xd2 += partial
 227
 228         XSTR    [dest1+pos], xd1
 229         XSTR    [dest2+pos], xd2
 230         XSTR    [dest3+pos], xd3
 231         XSTR    [dest4+pos], xd4
 232
 233         add     pos, 64                 ;Loop on 64 bytes at a time
 234         cmp     pos, len
 235         jle     .loop64
 236
 237         lea     tmp, [len + 64]
 238         cmp     pos, tmp
 239         je      .return_pass
 240
 241         ;; Tail len
 242         mov     pos, (1 << 63)
 243         lea     tmp, [len + 64 - 1]
 244         and     tmp, 63
 245         sarx    pos, pos, tmp
 246         kmovq   k1, pos
 247         mov     pos, len        ;Overlapped offset length-64
 248         jmp     .loop64         ;Do one more overlap pass
 249
 250 .return_pass:
 251         mov     return, 0
 252         FUNC_RESTORE
 253         ret
 254
 255 .return_fail:
 256         mov     return, 1
 257         FUNC_RESTORE
 258         ret
 259
 260 endproc_frame
 261
 262 %else
 263 %ifidn __OUTPUT_FORMAT__, win64
 264 global no_gf_4vect_mad_avx512
 265 no_gf_4vect_mad_avx512:
 266 %endif
 267 %endif  ; ifdef HAVE_AS_KNOWS_AVX512