ceph/src/isa-l/erasure_code/gf_3vect_mad_avx512.asm

   1 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
   2 ;  Copyright(c) 2011-2015 Intel Corporation All rights reserved.
   3 ;
   4 ;  Redistribution and use in source and binary forms, with or without
   5 ;  modification, are permitted provided that the following conditions
   6 ;  are met:
   7 ;    * Redistributions of source code must retain the above copyright
   8 ;      notice, this list of conditions and the following disclaimer.
   9 ;    * Redistributions in binary form must reproduce the above copyright
  10 ;      notice, this list of conditions and the following disclaimer in
  11 ;      the documentation and/or other materials provided with the
  12 ;      distribution.
  13 ;    * Neither the name of Intel Corporation nor the names of its
  14 ;      contributors may be used to endorse or promote products derived
  15 ;      from this software without specific prior written permission.
  16 ;
  17 ;  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18 ;  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19 ;  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20 ;  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21 ;  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22 ;  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23 ;  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24 ;  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25 ;  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26 ;  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27 ;  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  29
  30 ;;;
  31 ;;; gf_3vect_mad_avx512(len, vec, vec_i, mul_array, src, dest);
  32 ;;;
  33
  34 %include "reg_sizes.asm"
  35
  36 %ifdef HAVE_AS_KNOWS_AVX512
  37
  38 %ifidn __OUTPUT_FORMAT__, elf64
  39  %define arg0   rdi
  40  %define arg1   rsi
  41  %define arg2   rdx
  42  %define arg3   rcx
  43  %define arg4   r8
  44  %define arg5   r9
  45  %define tmp    r11
  46  %define return rax
  47  %define func(x) x:
  48  %define FUNC_SAVE
  49  %define FUNC_RESTORE
  50 %endif
  51
  52 %ifidn __OUTPUT_FORMAT__, win64
  53  %define arg0   rcx
  54  %define arg1   rdx
  55  %define arg2   r8
  56  %define arg3   r9
  57  %define arg4   r12             ; must be saved, loaded and restored
  58  %define arg5   r15             ; must be saved and restored
  59  %define tmp    r11
  60  %define return rax
  61  %define return.w eax
  62  %define stack_size 16*10 + 3*8
  63  %define arg(x)      [rsp + stack_size + PS + PS*x]
  64  %define func(x) proc_frame x
  65
  66  %macro FUNC_SAVE 0
  67         sub     rsp, stack_size
  68         vmovdqa [rsp+16*0],xmm6
  69         vmovdqa [rsp+16*1],xmm7
  70         vmovdqa [rsp+16*2],xmm8
  71         vmovdqa [rsp+16*3],xmm9
  72         vmovdqa [rsp+16*4],xmm10
  73         vmovdqa [rsp+16*5],xmm11
  74         vmovdqa [rsp+16*6],xmm12
  75         vmovdqa [rsp+16*7],xmm13
  76         vmovdqa [rsp+16*8],xmm14
  77         vmovdqa [rsp+16*9],xmm15
  78         save_reg        r12,  10*16 + 0*8
  79         save_reg        r15,  10*16 + 1*8
  80         end_prolog
  81         mov     arg4, arg(4)
  82         mov     arg5, arg(5)
  83  %endmacro
  84
  85  %macro FUNC_RESTORE 0
  86         vmovdqa xmm6, [rsp+16*0]
  87         vmovdqa xmm7, [rsp+16*1]
  88         vmovdqa xmm8, [rsp+16*2]
  89         vmovdqa xmm9, [rsp+16*3]
  90         vmovdqa xmm10, [rsp+16*4]
  91         vmovdqa xmm11, [rsp+16*5]
  92         vmovdqa xmm12, [rsp+16*6]
  93         vmovdqa xmm13, [rsp+16*7]
  94         vmovdqa xmm14, [rsp+16*8]
  95         vmovdqa xmm15, [rsp+16*9]
  96         mov     r12,  [rsp + 10*16 + 0*8]
  97         mov     r15,  [rsp + 10*16 + 1*8]
  98         add     rsp, stack_size
  99  %endmacro
 100 %endif
 101
 102 %define PS    8
 103 %define len   arg0
 104 %define vec   arg1
 105 %define vec_i arg2
 106 %define mul_array arg3
 107 %define src   arg4
 108 %define dest1 arg5
 109 %define pos   return
 110 %define dest2 mul_array
 111 %define dest3 vec_i
 112
 113 %ifndef EC_ALIGNED_ADDR
 114 ;;; Use Un-aligned load/store
 115  %define XLDR vmovdqu8
 116  %define XSTR vmovdqu8
 117 %else
 118 ;;; Use Non-temporal load/stor
 119  %ifdef NO_NT_LDST
 120   %define XLDR vmovdqa
 121   %define XSTR vmovdqa
 122  %else
 123   %define XLDR vmovntdqa
 124   %define XSTR vmovntdq
 125  %endif
 126 %endif
 127
 128 default rel
 129 [bits 64]
 130 section .text
 131
 132 %define x0        zmm0
 133 %define xtmpa     zmm1
 134 %define xtmph1    zmm2
 135 %define xtmpl1    zmm3
 136 %define xtmph2    zmm4
 137 %define xtmpl2    zmm5
 138 %define xtmph3    zmm6
 139 %define xtmpl3    zmm7
 140 %define xgft1_hi  zmm8
 141 %define xgft1_lo  zmm9
 142 %define xgft1_loy ymm9
 143 %define xgft2_hi  zmm10
 144 %define xgft2_lo  zmm11
 145 %define xgft2_loy ymm11
 146 %define xgft3_hi  zmm12
 147 %define xgft3_lo  zmm13
 148 %define xgft3_loy ymm13
 149 %define xd1       zmm14
 150 %define xd2       zmm15
 151 %define xd3       zmm16
 152 %define xmask0f   zmm17
 153
 154 align 16
 155 global gf_3vect_mad_avx512:function
 156 func(gf_3vect_mad_avx512)
 157         FUNC_SAVE
 158         sub     len, 64
 159         jl      .return_fail
 160         xor     pos, pos
 161         mov     tmp, 0x0f
 162         vpbroadcastb xmask0f, tmp       ;Construct mask 0x0f0f0f...
 163         sal     vec_i, 5                ;Multiply by 32
 164         sal     vec, 5
 165         lea     tmp, [mul_array + vec_i]
 166         vmovdqu xgft1_loy, [tmp]        ;Load array Ax{00}..{0f}, Ax{00}..{f0}
 167         vmovdqu xgft2_loy, [tmp+vec]    ;Load array Bx{00}..{0f}, Bx{00}..{f0}
 168         vmovdqu xgft3_loy, [tmp+2*vec]  ;Load array Cx{00}..{0f}, Cx{00}..{f0}
 169         vshufi64x2 xgft1_hi, xgft1_lo, xgft1_lo, 0x55
 170         vshufi64x2 xgft1_lo, xgft1_lo, xgft1_lo, 0x00
 171         vshufi64x2 xgft2_hi, xgft2_lo, xgft2_lo, 0x55
 172         vshufi64x2 xgft2_lo, xgft2_lo, xgft2_lo, 0x00
 173         vshufi64x2 xgft3_hi, xgft3_lo, xgft3_lo, 0x55
 174         vshufi64x2 xgft3_lo, xgft3_lo, xgft3_lo, 0x00
 175         mov     dest2, [dest1+PS]       ; reuse mul_array
 176         mov     dest3, [dest1+2*PS]     ; reuse vec_i
 177         mov     dest1, [dest1]
 178         mov     tmp, -1
 179         kmovq   k1, tmp
 180
 181 .loop64:
 182         XLDR    x0, [src+pos]           ;Get next source vector
 183         XLDR    xd1, [dest1+pos]        ;Get next dest vector
 184         XLDR    xd2, [dest2+pos]        ;Get next dest vector
 185         XLDR    xd3, [dest3+pos]        ;Get next dest vector
 186
 187         vpandq  xtmpa, x0, xmask0f      ;Mask low src nibble in bits 4-0
 188         vpsraw  x0, x0, 4               ;Shift to put high nibble into bits 4-0
 189         vpandq  x0, x0, xmask0f         ;Mask high src nibble in bits 4-0
 190
 191         ; dest1
 192         vpshufb xtmph1 {k1}{z}, xgft1_hi, x0    ;Lookup mul table of high nibble
 193         vpshufb xtmpl1 {k1}{z}, xgft1_lo, xtmpa ;Lookup mul table of low nibble
 194         vpxorq  xtmph1, xtmph1, xtmpl1          ;GF add high and low partials
 195         vpxorq  xd1, xd1, xtmph1                ;xd1 += partial
 196
 197         ; dest2
 198         vpshufb xtmph2 {k1}{z}, xgft2_hi, x0    ;Lookup mul table of high nibble
 199         vpshufb xtmpl2 {k1}{z}, xgft2_lo, xtmpa ;Lookup mul table of low nibble
 200         vpxorq  xtmph2, xtmph2, xtmpl2          ;GF add high and low partials
 201         vpxorq  xd2, xd2, xtmph2                ;xd2 += partial
 202
 203         ; dest3
 204         vpshufb xtmph3 {k1}{z}, xgft3_hi, x0    ;Lookup mul table of high nibble
 205         vpshufb xtmpl3 {k1}{z}, xgft3_lo, xtmpa ;Lookup mul table of low nibble
 206         vpxorq  xtmph3, xtmph3, xtmpl3          ;GF add high and low partials
 207         vpxorq  xd3, xd3, xtmph3                ;xd2 += partial
 208
 209         XSTR    [dest1+pos], xd1
 210         XSTR    [dest2+pos], xd2
 211         XSTR    [dest3+pos], xd3
 212
 213         add     pos, 64                 ;Loop on 64 bytes at a time
 214         cmp     pos, len
 215         jle     .loop64
 216
 217         lea     tmp, [len + 64]
 218         cmp     pos, tmp
 219         je      .return_pass
 220
 221         ;; Tail len
 222         mov     pos, (1 << 63)
 223         lea     tmp, [len + 64 - 1]
 224         and     tmp, 63
 225         sarx    pos, pos, tmp
 226         kmovq   k1, pos
 227         mov     pos, len        ;Overlapped offset length-64
 228         jmp     .loop64         ;Do one more overlap pass
 229
 230 .return_pass:
 231         mov     return, 0
 232         FUNC_RESTORE
 233         ret
 234
 235 .return_fail:
 236         mov     return, 1
 237         FUNC_RESTORE
 238         ret
 239
 240 endproc_frame
 241
 242 %else
 243 %ifidn __OUTPUT_FORMAT__, win64
 244 global no_gf_3vect_mad_avx512
 245 no_gf_3vect_mad_avx512:
 246 %endif
 247 %endif  ; ifdef HAVE_AS_KNOWS_AVX512