ceph/src/isa-l/raid/pq_gen_sse.asm

   1 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
   2 ;  Copyright(c) 2011-2015 Intel Corporation All rights reserved.
   3 ;
   4 ;  Redistribution and use in source and binary forms, with or without
   5 ;  modification, are permitted provided that the following conditions
   6 ;  are met:
   7 ;    * Redistributions of source code must retain the above copyright
   8 ;      notice, this list of conditions and the following disclaimer.
   9 ;    * Redistributions in binary form must reproduce the above copyright
  10 ;      notice, this list of conditions and the following disclaimer in
  11 ;      the documentation and/or other materials provided with the
  12 ;      distribution.
  13 ;    * Neither the name of Intel Corporation nor the names of its
  14 ;      contributors may be used to endorse or promote products derived
  15 ;      from this software without specific prior written permission.
  16 ;
  17 ;  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18 ;  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19 ;  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20 ;  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21 ;  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22 ;  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23 ;  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24 ;  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25 ;  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26 ;  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27 ;  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  29
  30 ;;; Optimized pq of N source vectors using SSE3
  31 ;;; int pq_gen_sse(int vects, int len, void **array)
  32
  33 ;;; Generates P+Q parity vector from N (vects-2) sources in array of pointers
  34 ;;; (**array).  Last two pointers are the P and Q destinations respectively.
  35 ;;; Vectors must be aligned to 16 bytes.  Length must be 16 byte aligned.
  36
  37 %include "reg_sizes.asm"
  38
  39 %ifidn __OUTPUT_FORMAT__, elf64
  40  %define arg0  rdi
  41  %define arg1  rsi
  42  %define arg2  rdx
  43  %define arg3  rcx
  44  %define arg4  r8
  45  %define arg5  r9
  46  %define tmp   r11
  47  %define tmp3  arg4
  48  %define return rax
  49  %define func(x) x:
  50  %define FUNC_SAVE
  51  %define FUNC_RESTORE
  52 %endif
  53
  54 %ifidn __OUTPUT_FORMAT__, win64
  55  %define arg0  rcx
  56  %define arg1  rdx
  57  %define arg2  r8
  58  %define arg3  r9
  59  %define tmp   r11
  60  %define tmp3  r10
  61  %define return rax
  62  %define stack_size  7*16 + 8   ; must be an odd multiple of 8
  63  %define func(x) proc_frame x
  64  %macro FUNC_SAVE 0
  65         alloc_stack     stack_size
  66         save_xmm128     xmm6, 0*16
  67         save_xmm128     xmm7, 1*16
  68         save_xmm128     xmm8, 2*16
  69         save_xmm128     xmm9, 3*16
  70         save_xmm128     xmm10, 4*16
  71         save_xmm128     xmm11, 5*16
  72         save_xmm128     xmm15, 6*16
  73         end_prolog
  74  %endmacro
  75
  76  %macro FUNC_RESTORE 0
  77         movdqa  xmm6, [rsp + 0*16]
  78         movdqa  xmm7, [rsp + 1*16]
  79         movdqa  xmm8, [rsp + 2*16]
  80         movdqa  xmm9, [rsp + 3*16]
  81         movdqa  xmm10, [rsp + 4*16]
  82         movdqa  xmm11, [rsp + 5*16]
  83         movdqa  xmm15, [rsp + 6*16]
  84         add     rsp, stack_size
  85  %endmacro
  86 %endif
  87
  88 %define vec arg0
  89 %define len arg1
  90 %define ptr arg3
  91 %define pos rax
  92
  93 %define xp1   xmm0
  94 %define xq1   xmm1
  95 %define xtmp1 xmm2
  96 %define xs1   xmm3
  97
  98 %define xp2   xmm4
  99 %define xq2   xmm5
 100 %define xtmp2 xmm6
 101 %define xs2   xmm7
 102
 103 %define xp3   xmm8
 104 %define xq3   xmm9
 105 %define xtmp3 xmm10
 106 %define xs3   xmm11
 107
 108 %define xpoly xmm15
 109
 110 ;;; Use Non-temporal load/stor
 111 %ifdef NO_NT_LDST
 112  %define XLDR movdqa
 113  %define XSTR movdqa
 114 %else
 115  %define XLDR movntdqa
 116  %define XSTR movntdq
 117 %endif
 118
 119 default rel
 120
 121 [bits 64]
 122 section .text
 123
 124 align 16
 125 global pq_gen_sse:function
 126 func(pq_gen_sse)
 127         FUNC_SAVE
 128         sub     vec, 3                  ;Keep as offset to last source
 129         jng     return_fail             ;Must have at least 2 sources
 130         cmp     len, 0
 131         je      return_pass
 132         test    len, (16-1)             ;Check alignment of length
 133         jnz     return_fail
 134         mov     pos, 0
 135         movdqa  xpoly, [poly]
 136         cmp     len, 48
 137         jl      loop16
 138
 139 len_aligned_32bytes:
 140         sub     len, 48                 ;Len points to last block
 141
 142 loop48:
 143         mov     ptr, [arg2+vec*8]       ;Fetch last source pointer
 144         mov     tmp, vec                ;Set tmp to point back to last vector
 145         XLDR    xs1, [ptr+pos]          ;Preload last vector (source)
 146         XLDR    xs2, [ptr+pos+16]       ;Preload last vector (source)
 147         XLDR    xs3, [ptr+pos+32]       ;Preload last vector (source)
 148         pxor    xp1, xp1                ;p1 = 0
 149         pxor    xp2, xp2                ;p2 = 0
 150         pxor    xp3, xp3                ;p3 = 0
 151         pxor    xq1, xq1                ;q1 = 0
 152         pxor    xq2, xq2                ;q2 = 0
 153         pxor    xq3, xq3                ;q3 = 0
 154
 155 next_vect:
 156         sub     tmp, 1                  ;Inner loop for each source vector
 157         mov     ptr, [arg2+tmp*8]       ; get pointer to next vect
 158         pxor    xq1, xs1                ; q1 ^= s1
 159         pxor    xq2, xs2                ; q2 ^= s2
 160         pxor    xq3, xs3                ; q3 ^= s3
 161         pxor    xp1, xs1                ; p1 ^= s1
 162         pxor    xp2, xs2                ; p2 ^= s2
 163         pxor    xp3, xs3                ; p3 ^= s2
 164         pxor    xtmp1, xtmp1            ; xtmp1 = 0 - for compare to 0
 165         pxor    xtmp2, xtmp2            ; xtmp2 = 0
 166         pxor    xtmp3, xtmp3            ; xtmp3 = 0
 167         pcmpgtb xtmp1, xq1              ; xtmp1 = mask 0xff or 0x00 if bit7 set
 168         pcmpgtb xtmp2, xq2              ; xtmp2 = mask 0xff or 0x00 if bit7 set
 169         pcmpgtb xtmp3, xq3              ; xtmp3 = mask 0xff or 0x00 if bit7 set
 170         pand    xtmp1, xpoly            ; xtmp1 = poly or 0x00
 171         pand    xtmp2, xpoly            ; xtmp2 = poly or 0x00
 172         pand    xtmp3, xpoly            ; xtmp3 = poly or 0x00
 173         XLDR    xs1, [ptr+pos]          ; Get next vector (source data1)
 174         XLDR    xs2, [ptr+pos+16]       ; Get next vector (source data2)
 175         XLDR    xs3, [ptr+pos+32]       ; Get next vector (source data3)
 176         paddb   xq1, xq1                ; q1 = q1<<1
 177         paddb   xq2, xq2                ; q2 = q2<<1
 178         paddb   xq3, xq3                ; q3 = q3<<1
 179         pxor    xq1, xtmp1              ; q1 = q1<<1 ^ poly_masked
 180         pxor    xq2, xtmp2              ; q2 = q2<<1 ^ poly_masked
 181         pxor    xq3, xtmp3              ; q3 = q3<<1 ^ poly_masked
 182         jg      next_vect               ; Loop for each vect except 0
 183
 184         mov     ptr, [arg2+8+vec*8]     ;Get address of P parity vector
 185         mov     tmp, [arg2+(2*8)+vec*8] ;Get address of Q parity vector
 186         pxor    xp1, xs1                ;p1 ^= s1[0] - last source is already loaded
 187         pxor    xq1, xs1                ;q1 ^= 1 * s1[0]
 188         pxor    xp2, xs2                ;p2 ^= s2[0]
 189         pxor    xq2, xs2                ;q2 ^= 1 * s2[0]
 190         pxor    xp3, xs3                ;p3 ^= s3[0]
 191         pxor    xq3, xs3                ;q3 ^= 1 * s3[0]
 192         XSTR    [ptr+pos], xp1          ;Write parity P1 vector
 193         XSTR    [ptr+pos+16], xp2       ;Write parity P2 vector
 194         XSTR    [ptr+pos+32], xp3       ;Write parity P3 vector
 195         XSTR    [tmp+pos], xq1          ;Write parity Q1 vector
 196         XSTR    [tmp+pos+16], xq2       ;Write parity Q2 vector
 197         XSTR    [tmp+pos+32], xq3       ;Write parity Q3 vector
 198         add     pos, 48
 199         cmp     pos, len
 200         jle     loop48
 201
 202         ;; ------------------------------
 203         ;; Do last 16 or 32 Bytes remaining
 204         add     len, 48
 205         cmp     pos, len
 206         je      return_pass
 207
 208 loop16:
 209         mov     ptr, [arg2+vec*8]       ;Fetch last source pointer
 210         mov     tmp, vec                ;Set tmp to point back to last vector
 211         XLDR    xs1, [ptr+pos]          ;Preload last vector (source)
 212         pxor    xp1, xp1                ;p = 0
 213         pxor    xq1, xq1                ;q = 0
 214
 215 next_vect16:
 216         sub     tmp, 1                  ;Inner loop for each source vector
 217         mov     ptr, [arg2+tmp*8]       ; get pointer to next vect
 218         pxor    xq1, xs1                ; q1 ^= s1
 219         pxor    xtmp1, xtmp1            ; xtmp = 0
 220         pcmpgtb xtmp1, xq1              ; xtmp = mask 0xff or 0x00 if bit7 set
 221         pand    xtmp1, xpoly            ; xtmp = poly or 0x00
 222         pxor    xp1, xs1                ; p ^= s
 223         paddb   xq1, xq1                ; q = q<<1
 224         pxor    xq1, xtmp1              ; q = q<<1 ^ poly_masked
 225         XLDR    xs1, [ptr+pos]          ; Get next vector (source data)
 226         jg      next_vect16             ; Loop for each vect except 0
 227
 228         mov     ptr, [arg2+8+vec*8]     ;Get address of P parity vector
 229         mov     tmp, [arg2+(2*8)+vec*8] ;Get address of Q parity vector
 230         pxor    xp1, xs1                ;p ^= s[0] - last source is already loaded
 231         pxor    xq1, xs1                ;q ^= 1 * s[0]
 232         XSTR    [ptr+pos], xp1          ;Write parity P vector
 233         XSTR    [tmp+pos], xq1          ;Write parity Q vector
 234         add     pos, 16
 235         cmp     pos, len
 236         jl      loop16
 237
 238
 239 return_pass:
 240         mov     return, 0
 241         FUNC_RESTORE
 242         ret
 243
 244 return_fail:
 245         mov     return, 1
 246         FUNC_RESTORE
 247         ret
 248
 249 endproc_frame
 250
 251 section .data
 252
 253 align 16
 254 poly:
 255 dq 0x1d1d1d1d1d1d1d1d, 0x1d1d1d1d1d1d1d1d
 256
 257 ;;;       func        core, ver, snum
 258 slversion pq_gen_sse, 00,   09,  0032