ceph/src/spdk/isa-l/raid/pq_check_sse.asm

   1 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
   2 ;  Copyright(c) 2011-2015 Intel Corporation All rights reserved.
   3 ;
   4 ;  Redistribution and use in source and binary forms, with or without
   5 ;  modification, are permitted provided that the following conditions
   6 ;  are met:
   7 ;    * Redistributions of source code must retain the above copyright
   8 ;      notice, this list of conditions and the following disclaimer.
   9 ;    * Redistributions in binary form must reproduce the above copyright
  10 ;      notice, this list of conditions and the following disclaimer in
  11 ;      the documentation and/or other materials provided with the
  12 ;      distribution.
  13 ;    * Neither the name of Intel Corporation nor the names of its
  14 ;      contributors may be used to endorse or promote products derived
  15 ;      from this software without specific prior written permission.
  16 ;
  17 ;  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18 ;  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19 ;  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20 ;  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21 ;  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22 ;  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23 ;  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24 ;  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25 ;  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26 ;  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27 ;  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  29
  30 ;;; Optimized pq of N source vectors using SSE3
  31 ;;; int pq_check_sse(int vects, int len, void **array)
  32
  33 ;;; Generates P+Q parity vector from N (vects-2) sources in array of pointers
  34 ;;; (**array).  Last two pointers are the P and Q destinations respectively.
  35 ;;; Vectors must be aligned to 16 bytes.  Length must be 16 byte aligned.
  36
  37 %include "reg_sizes.asm"
  38
  39 %ifidn __OUTPUT_FORMAT__, elf64
  40  %define arg0  rdi
  41  %define arg1  rsi
  42  %define arg2  rdx
  43  %define arg3  rcx
  44  %define arg4  r8
  45  %define arg5  r9
  46  %define tmp   r11
  47  %define tmp3  arg4
  48  %define return rax
  49  %define func(x) x:
  50  %define FUNC_SAVE
  51  %define FUNC_RESTORE
  52 %endif
  53
  54 %ifidn __OUTPUT_FORMAT__, win64
  55  %define arg0  rcx
  56  %define arg1  rdx
  57  %define arg2  r8
  58  %define arg3  r9
  59  %define tmp   r11
  60  %define tmp3  r10
  61  %define return rax
  62  %define stack_size  7*16 + 8   ; must be an odd multiple of 8
  63  %define func(x) proc_frame x
  64  %macro FUNC_SAVE 0
  65         alloc_stack     stack_size
  66         save_xmm128     xmm6, 0*16
  67         save_xmm128     xmm7, 1*16
  68         save_xmm128     xmm8, 2*16
  69         save_xmm128     xmm9, 3*16
  70         save_xmm128     xmm10, 4*16
  71         save_xmm128     xmm11, 5*16
  72         save_xmm128     xmm15, 6*16
  73         end_prolog
  74  %endmacro
  75
  76  %macro FUNC_RESTORE 0
  77         movdqa  xmm6, [rsp + 0*16]
  78         movdqa  xmm7, [rsp + 1*16]
  79         movdqa  xmm8, [rsp + 2*16]
  80         movdqa  xmm9, [rsp + 3*16]
  81         movdqa  xmm10, [rsp + 4*16]
  82         movdqa  xmm11, [rsp + 5*16]
  83         movdqa  xmm15, [rsp + 9*16]
  84         add     rsp, stack_size
  85  %endmacro
  86 %endif
  87
  88 %define vec arg0
  89 %define len arg1
  90 %define ptr arg3
  91 %define pos return
  92
  93 %define xp1   xmm0
  94 %define xq1   xmm1
  95 %define xtmp1 xmm2
  96 %define xs1   xmm3
  97
  98 %define xp2   xmm4
  99 %define xq2   xmm5
 100 %define xtmp2 xmm6
 101 %define xs2   xmm7
 102
 103 %define xp3   xmm8
 104 %define xq3   xmm9
 105 %define xtmp3 xmm10
 106 %define xs3   xmm11
 107
 108 %define xpoly xmm15
 109
 110 ;;; Use Non-temporal load/stor
 111 %ifdef NO_NT_LDST
 112  %define XLDR movdqa
 113  %define XSTR movdqa
 114 %else
 115  %define XLDR movdqa
 116  %define XSTR movntdq
 117 %endif
 118
 119 default rel
 120
 121 [bits 64]
 122 section .text
 123
 124 align 16
 125 global pq_check_sse:ISAL_SYM_TYPE_FUNCTION
 126 func(pq_check_sse)
 127         FUNC_SAVE
 128         sub     vec, 3                  ;Keep as offset to last source
 129         jng     return_fail             ;Must have at least 2 sources
 130         cmp     len, 0
 131         je      return_pass
 132         test    len, (16-1)             ;Check alignment of length
 133         jnz     return_fail
 134         mov     pos, 0
 135         movdqa  xpoly, [poly]
 136         cmp     len, 48
 137         jl      loop16
 138
 139 len_aligned_32bytes:
 140         sub     len, 48                 ;Do end of vec first and run backward
 141
 142 loop48:
 143         mov     ptr, [arg2+8+vec*8]     ;Get address of P parity vector
 144         mov     tmp, [arg2+(2*8)+vec*8] ;Get address of Q parity vector
 145         XLDR    xp1, [ptr+pos]          ;Initialize xp1 with P1 src
 146         XLDR    xp2, [ptr+pos+16]       ;Initialize xp2 with P2 src + 16B ahead
 147         XLDR    xp3, [ptr+pos+32]       ;Initialize xp3 with P2 src + 32B ahead
 148         pxor    xq1, xq1                ;q1 = 0
 149         pxor    xq2, xq2                ;q2 = 0
 150         pxor    xq3, xq3                ;q3 = 0
 151
 152         mov     ptr, [arg2+vec*8]       ;Fetch last source pointer
 153         mov     tmp, vec                ;Set tmp to point back to last vector
 154         XLDR    xs1, [ptr+pos]          ;Preload last vector (source)
 155         XLDR    xs2, [ptr+pos+16]       ;Preload last vector (source)
 156         XLDR    xs3, [ptr+pos+32]       ;Preload last vector (source)
 157
 158 next_vect:
 159         sub     tmp, 1                  ;Inner loop for each source vector
 160         mov     ptr, [arg2+tmp*8]       ; get pointer to next vect
 161         pxor    xp1, xs1                ; p1 ^= s1
 162         pxor    xp2, xs2                ; p2 ^= s2
 163         pxor    xp3, xs3                ; p3 ^= s2
 164         pxor    xq1, xs1                ; q1 ^= s1
 165         pxor    xq2, xs2                ; q2 ^= s2
 166         pxor    xq3, xs3                ; q3 ^= s3
 167         pxor    xtmp1, xtmp1            ; xtmp1 = 0 - for compare to 0
 168         pxor    xtmp2, xtmp2            ; xtmp2 = 0
 169         pxor    xtmp3, xtmp3            ; xtmp3 = 0
 170         pcmpgtb xtmp1, xq1              ; xtmp1 = mask 0xff or 0x00 if bit7 set
 171         pcmpgtb xtmp2, xq2              ; xtmp2 = mask 0xff or 0x00 if bit7 set
 172         pcmpgtb xtmp3, xq3              ; xtmp3 = mask 0xff or 0x00 if bit7 set
 173         pand    xtmp1, xpoly            ; xtmp1 = poly or 0x00
 174         pand    xtmp2, xpoly            ; xtmp2 = poly or 0x00
 175         pand    xtmp3, xpoly            ; xtmp3 = poly or 0x00
 176         XLDR    xs1, [ptr+pos]          ; Get next vector (source data1)
 177         XLDR    xs2, [ptr+pos+16]       ; Get next vector (source data2)
 178         XLDR    xs3, [ptr+pos+32]       ; Get next vector (source data3)
 179         paddb   xq1, xq1                ; q1 = q1<<1
 180         paddb   xq2, xq2                ; q2 = q2<<1
 181         paddb   xq3, xq3                ; q3 = q3<<1
 182         pxor    xq1, xtmp1              ; q1 = q1<<1 ^ poly_masked
 183         pxor    xq2, xtmp2              ; q2 = q2<<1 ^ poly_masked
 184         pxor    xq3, xtmp3              ; q3 = q3<<1 ^ poly_masked
 185         jg      next_vect               ; Loop for each vect except 0
 186
 187         pxor    xp1, xs1                ;p1 ^= s1[0] - last source is already loaded
 188         pxor    xq1, xs1                ;q1 ^= 1 * s1[0]
 189         pxor    xp2, xs2                ;p2 ^= s2[0]
 190         pxor    xq2, xs2                ;q2 ^= 1 * s2[0]
 191         pxor    xp3, xs3                ;p3 ^= s3[0]
 192         pxor    xq3, xs3                ;q3 ^= 1 * s3[0]
 193
 194         mov     tmp, [arg2+(2*8)+vec*8] ;Get address of Q parity vector
 195         XLDR    xtmp1, [tmp+pos]        ;re-init xq1 with Q1 src
 196         XLDR    xtmp2, [tmp+pos+16]     ;re-init xq2 with Q2 src + 16B ahead
 197         XLDR    xtmp3, [tmp+pos+32]     ;re-init xq3 with Q2 src + 32B ahead
 198
 199         pxor    xq1, xtmp1              ;xq1 = q1 calculated ^ q1 saved
 200         pxor    xq2, xtmp2
 201         pxor    xq3, xtmp3
 202
 203         por     xp1, xq1                ;Confirm that all P&Q parity are 0
 204         por     xp1, xp2
 205         por     xp1, xq2
 206         por     xp1, xp3
 207         por     xp1, xq3
 208         ptest   xp1, xp1
 209         jnz     return_fail
 210         add     pos, 48
 211         cmp     pos, len
 212         jle     loop48
 213
 214
 215         ;; ------------------------------
 216         ;; Do last 16 or 32 Bytes remaining
 217         add     len, 48
 218         cmp     pos, len
 219         je      return_pass
 220
 221 loop16:
 222         mov     ptr, [arg2+8+vec*8]     ;Get address of P parity vector
 223         mov     tmp, [arg2+(2*8)+vec*8] ;Get address of Q parity vector
 224         XLDR    xp1, [ptr+pos]          ;Initialize xp1 with P1 src
 225         pxor    xq1, xq1                ;q = 0
 226         mov     ptr, [arg2+vec*8]       ;Fetch last source pointer
 227         mov     tmp, vec                ;Set tmp to point back to last vector
 228         XLDR    xs1, [ptr+pos]          ;Preload last vector (source)
 229
 230 next_vect16:
 231         sub     tmp, 1                  ;Inner loop for each source vector
 232         mov     ptr, [arg2+tmp*8]       ; get pointer to next vect
 233         pxor    xq1, xs1                ; q ^= s
 234         pxor    xtmp1, xtmp1            ; xtmp = 0
 235         pcmpgtb xtmp1, xq1              ; xtmp = mask 0xff or 0x00 if bit7 set
 236         pand    xtmp1, xpoly            ; xtmp = poly or 0x00
 237         pxor    xp1, xs1                ; p ^= s
 238         paddb   xq1, xq1                ; q = q<<1
 239         pxor    xq1, xtmp1              ; q = q<<1 ^ poly_masked
 240         XLDR    xs1, [ptr+pos]          ; Get next vector (source data)
 241         jg      next_vect16             ; Loop for each vect except 0
 242
 243         pxor    xp1, xs1                ;p ^= s[0] - last source is already loaded
 244         pxor    xq1, xs1                ;q ^= 1 * s[0]
 245
 246         mov     tmp, [arg2+(2*8)+vec*8] ;Get address of Q parity vector
 247         XLDR    xtmp1, [tmp+pos]        ;re-init tmp with Q1 src
 248         pxor    xq1, xtmp1              ;xq1 = q1 calculated ^ q1 saved
 249
 250         por     xp1, xq1                ;Confirm that all P&Q parity are = 0
 251         ptest   xp1, xp1
 252         jnz     return_fail
 253         add     pos, 16
 254         cmp     pos, len
 255         jl      loop16
 256
 257
 258 return_pass:
 259         mov     return, 0
 260         FUNC_RESTORE
 261         ret
 262
 263 return_fail:
 264         mov     return, 1
 265         FUNC_RESTORE
 266         ret
 267
 268 endproc_frame
 269
 270 section .data
 271
 272 align 16
 273 poly:
 274 dq 0x1d1d1d1d1d1d1d1d, 0x1d1d1d1d1d1d1d1d
 275
 276 ;;;       func          core, ver, snum
 277 slversion pq_check_sse, 00,   06,  0033