ceph/src/isa-l/erasure_code/gf_4vect_dot_prod_avx512.asm

   1 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
   2 ;  Copyright(c) 2011-2015 Intel Corporation All rights reserved.
   3 ;
   4 ;  Redistribution and use in source and binary forms, with or without
   5 ;  modification, are permitted provided that the following conditions
   6 ;  are met:
   7 ;    * Redistributions of source code must retain the above copyright
   8 ;      notice, this list of conditions and the following disclaimer.
   9 ;    * Redistributions in binary form must reproduce the above copyright
  10 ;      notice, this list of conditions and the following disclaimer in
  11 ;      the documentation and/or other materials provided with the
  12 ;      distribution.
  13 ;    * Neither the name of Intel Corporation nor the names of its
  14 ;      contributors may be used to endorse or promote products derived
  15 ;      from this software without specific prior written permission.
  16 ;
  17 ;  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18 ;  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19 ;  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20 ;  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21 ;  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22 ;  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23 ;  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24 ;  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25 ;  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26 ;  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27 ;  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  29
  30 ;;;
  31 ;;; gf_4vect_dot_prod_avx512(len, vec, *g_tbls, **buffs, **dests);
  32 ;;;
  33
  34 %include "reg_sizes.asm"
  35
  36 %ifdef HAVE_AS_KNOWS_AVX512
  37
  38 %ifidn __OUTPUT_FORMAT__, elf64
  39  %define arg0  rdi
  40  %define arg1  rsi
  41  %define arg2  rdx
  42  %define arg3  rcx
  43  %define arg4  r8
  44  %define arg5  r9
  45
  46  %define tmp   r11
  47  %define tmp.w r11d
  48  %define tmp.b r11b
  49  %define tmp2  r10
  50  %define tmp3  r13              ; must be saved and restored
  51  %define tmp4  r12              ; must be saved and restored
  52  %define tmp5  r14              ; must be saved and restored
  53  %define tmp6  r15              ; must be saved and restored
  54  %define return rax
  55  %define PS     8
  56  %define LOG_PS 3
  57
  58  %define func(x) x:
  59  %macro FUNC_SAVE 0
  60         push    r12
  61         push    r13
  62         push    r14
  63         push    r15
  64  %endmacro
  65  %macro FUNC_RESTORE 0
  66         pop     r15
  67         pop     r14
  68         pop     r13
  69         pop     r12
  70  %endmacro
  71 %endif
  72
  73 %ifidn __OUTPUT_FORMAT__, win64
  74  %define arg0   rcx
  75  %define arg1   rdx
  76  %define arg2   r8
  77  %define arg3   r9
  78
  79  %define arg4   r12             ; must be saved, loaded and restored
  80  %define arg5   r15             ; must be saved and restored
  81  %define tmp    r11
  82  %define tmp.w  r11d
  83  %define tmp.b  r11b
  84  %define tmp2   r10
  85  %define tmp3   r13             ; must be saved and restored
  86  %define tmp4   r14             ; must be saved and restored
  87  %define tmp5   rdi             ; must be saved and restored
  88  %define tmp6   rsi             ; must be saved and restored
  89  %define return rax
  90  %define PS     8
  91  %define LOG_PS 3
  92  %define stack_size  9*16 + 7*8         ; must be an odd multiple of 8
  93  %define arg(x)      [rsp + stack_size + PS + PS*x]
  94
  95  %define func(x) proc_frame x
  96  %macro FUNC_SAVE 0
  97         alloc_stack     stack_size
  98         vmovdqa [rsp + 0*16], xmm6
  99         vmovdqa [rsp + 1*16], xmm7
 100         vmovdqa [rsp + 2*16], xmm8
 101         vmovdqa [rsp + 3*16], xmm9
 102         vmovdqa [rsp + 4*16], xmm10
 103         vmovdqa [rsp + 5*16], xmm11
 104         vmovdqa [rsp + 6*16], xmm12
 105         vmovdqa [rsp + 7*16], xmm13
 106         vmovdqa [rsp + 8*16], xmm14
 107         save_reg        r12,  9*16 + 0*8
 108         save_reg        r13,  9*16 + 1*8
 109         save_reg        r14,  9*16 + 2*8
 110         save_reg        r15,  9*16 + 3*8
 111         save_reg        rdi,  9*16 + 4*8
 112         save_reg        rsi,  9*16 + 5*8
 113         end_prolog
 114         mov     arg4, arg(4)
 115  %endmacro
 116
 117  %macro FUNC_RESTORE 0
 118         vmovdqa xmm6, [rsp + 0*16]
 119         vmovdqa xmm7, [rsp + 1*16]
 120         vmovdqa xmm8, [rsp + 2*16]
 121         vmovdqa xmm9, [rsp + 3*16]
 122         vmovdqa xmm10, [rsp + 4*16]
 123         vmovdqa xmm11, [rsp + 5*16]
 124         vmovdqa xmm12, [rsp + 6*16]
 125         vmovdqa xmm13, [rsp + 7*16]
 126         vmovdqa xmm14, [rsp + 8*16]
 127         mov     r12,  [rsp + 9*16 + 0*8]
 128         mov     r13,  [rsp + 9*16 + 1*8]
 129         mov     r14,  [rsp + 9*16 + 2*8]
 130         mov     r15,  [rsp + 9*16 + 3*8]
 131         mov     rdi,  [rsp + 9*16 + 4*8]
 132         mov     rsi,  [rsp + 9*16 + 5*8]
 133         add     rsp, stack_size
 134  %endmacro
 135 %endif
 136
 137
 138 %define len    arg0
 139 %define vec    arg1
 140 %define mul_array arg2
 141 %define src    arg3
 142 %define dest1  arg4
 143 %define ptr    arg5
 144 %define vec_i  tmp2
 145 %define dest2  tmp3
 146 %define dest3  tmp4
 147 %define dest4  tmp5
 148 %define vskip3 tmp6
 149 %define pos    return
 150
 151
 152 %ifndef EC_ALIGNED_ADDR
 153 ;;; Use Un-aligned load/store
 154  %define XLDR vmovdqu8
 155  %define XSTR vmovdqu8
 156 %else
 157 ;;; Use Non-temporal load/stor
 158  %ifdef NO_NT_LDST
 159   %define XLDR vmovdqa
 160   %define XSTR vmovdqa
 161  %else
 162   %define XLDR vmovntdqa
 163   %define XSTR vmovntdq
 164  %endif
 165 %endif
 166
 167 %define xmask0f   zmm14
 168 %define xgft1_lo  zmm13
 169 %define xgft1_loy ymm13
 170 %define xgft1_hi  zmm12
 171 %define xgft2_lo  zmm11
 172 %define xgft2_loy ymm11
 173 %define xgft2_hi  zmm10
 174 %define xgft3_lo  zmm9
 175 %define xgft3_loy ymm9
 176 %define xgft3_hi  zmm8
 177 %define xgft4_lo  zmm7
 178 %define xgft4_loy ymm7
 179 %define xgft4_hi  zmm6
 180
 181 %define x0        zmm0
 182 %define xtmpa     zmm1
 183 %define xp1       zmm2
 184 %define xp2       zmm3
 185 %define xp3       zmm4
 186 %define xp4       zmm5
 187
 188 default rel
 189 [bits 64]
 190
 191 section .text
 192
 193 align 16
 194 global gf_4vect_dot_prod_avx512:function
 195 func(gf_4vect_dot_prod_avx512)
 196         FUNC_SAVE
 197         sub     len, 64
 198         jl      .return_fail
 199
 200         xor     pos, pos
 201         mov     tmp, 0x0f
 202         vpbroadcastb xmask0f, tmp       ;Construct mask 0x0f0f0f...
 203         mov     vskip3, vec
 204         imul    vskip3, 96
 205         sal     vec, LOG_PS             ;vec *= PS. Make vec_i count by PS
 206         mov     dest2, [dest1+PS]
 207         mov     dest3, [dest1+2*PS]
 208         mov     dest4, [dest1+3*PS]
 209         mov     dest1, [dest1]
 210
 211 .loop64:
 212         vpxorq  xp1, xp1, xp1
 213         vpxorq  xp2, xp2, xp2
 214         vpxorq  xp3, xp3, xp3
 215         vpxorq  xp4, xp4, xp4
 216         mov     tmp, mul_array
 217         xor     vec_i, vec_i
 218
 219 .next_vect:
 220         mov     ptr, [src+vec_i]
 221         XLDR    x0, [ptr+pos]           ;Get next source vector
 222         add     vec_i, PS
 223
 224         vpandq  xtmpa, x0, xmask0f      ;Mask low src nibble in bits 4-0
 225         vpsraw  x0, x0, 4               ;Shift to put high nibble into bits 4-0
 226         vpandq  x0, x0, xmask0f         ;Mask high src nibble in bits 4-0
 227
 228         vmovdqu8 xgft1_loy, [tmp]               ;Load array Ax{00}..{0f}, Ax{00}..{f0}
 229         vmovdqu8 xgft2_loy, [tmp+vec*(32/PS)]   ;Load array Bx{00}..{0f}, Bx{00}..{f0}
 230         vmovdqu8 xgft3_loy, [tmp+vec*(64/PS)]   ;Load array Cx{00}..{0f}, Cx{00}..{f0}
 231         vmovdqu8 xgft4_loy, [tmp+vskip3]        ;Load array Dx{00}..{0f}, Dx{00}..{f0}
 232         add     tmp, 32
 233
 234         vshufi64x2 xgft1_hi, xgft1_lo, xgft1_lo, 0x55
 235         vshufi64x2 xgft1_lo, xgft1_lo, xgft1_lo, 0x00
 236         vshufi64x2 xgft2_hi, xgft2_lo, xgft2_lo, 0x55
 237         vshufi64x2 xgft2_lo, xgft2_lo, xgft2_lo, 0x00
 238
 239         vpshufb xgft1_hi, xgft1_hi, x0          ;Lookup mul table of high nibble
 240         vpshufb xgft1_lo, xgft1_lo, xtmpa       ;Lookup mul table of low nibble
 241         vpxorq  xgft1_hi, xgft1_hi, xgft1_lo    ;GF add high and low partials
 242         vpxorq  xp1, xp1, xgft1_hi              ;xp1 += partial
 243
 244         vpshufb xgft2_hi, xgft2_hi, x0          ;Lookup mul table of high nibble
 245         vpshufb xgft2_lo, xgft2_lo, xtmpa       ;Lookup mul table of low nibble
 246         vpxorq  xgft2_hi, xgft2_hi, xgft2_lo    ;GF add high and low partials
 247         vpxorq  xp2, xp2, xgft2_hi              ;xp2 += partial
 248
 249         vshufi64x2 xgft3_hi, xgft3_lo, xgft3_lo, 0x55
 250         vshufi64x2 xgft3_lo, xgft3_lo, xgft3_lo, 0x00
 251         vshufi64x2 xgft4_hi, xgft4_lo, xgft4_lo, 0x55
 252         vshufi64x2 xgft4_lo, xgft4_lo, xgft4_lo, 0x00
 253
 254         vpshufb xgft3_hi, xgft3_hi, x0          ;Lookup mul table of high nibble
 255         vpshufb xgft3_lo, xgft3_lo, xtmpa       ;Lookup mul table of low nibble
 256         vpxorq  xgft3_hi, xgft3_hi, xgft3_lo    ;GF add high and low partials
 257         vpxorq  xp3, xp3, xgft3_hi              ;xp3 += partial
 258
 259         vpshufb xgft4_hi, xgft4_hi, x0          ;Lookup mul table of high nibble
 260         vpshufb xgft4_lo, xgft4_lo, xtmpa       ;Lookup mul table of low nibble
 261         vpxorq  xgft4_hi, xgft4_hi, xgft4_lo    ;GF add high and low partials
 262         vpxorq  xp4, xp4, xgft4_hi              ;xp4 += partial
 263
 264         cmp     vec_i, vec
 265         jl      .next_vect
 266
 267         XSTR    [dest1+pos], xp1
 268         XSTR    [dest2+pos], xp2
 269         XSTR    [dest3+pos], xp3
 270         XSTR    [dest4+pos], xp4
 271
 272         add     pos, 64                 ;Loop on 64 bytes at a time
 273         cmp     pos, len
 274         jle     .loop64
 275
 276         lea     tmp, [len + 64]
 277         cmp     pos, tmp
 278         je      .return_pass
 279
 280         ;; Tail len
 281         mov     pos, len        ;Overlapped offset length-64
 282         jmp     .loop64         ;Do one more overlap pass
 283
 284 .return_pass:
 285         mov     return, 0
 286         FUNC_RESTORE
 287         ret
 288
 289 .return_fail:
 290         mov     return, 1
 291         FUNC_RESTORE
 292         ret
 293
 294 endproc_frame
 295
 296 %else
 297 %ifidn __OUTPUT_FORMAT__, win64
 298 global no_gf_4vect_dot_prod_avx512
 299 no_gf_4vect_dot_prod_avx512:
 300 %endif
 301 %endif  ; ifdef HAVE_AS_KNOWS_AVX512