ceph/src/spdk/intel-ipsec-mb/sse/mb_mgr_hmac_sha_512_submit_sse.asm

   1 ;;
   2 ;; Copyright (c) 2012-2018, Intel Corporation
   3 ;;
   4 ;; Redistribution and use in source and binary forms, with or without
   5 ;; modification, are permitted provided that the following conditions are met:
   6 ;;
   7 ;;     * Redistributions of source code must retain the above copyright notice,
   8 ;;       this list of conditions and the following disclaimer.
   9 ;;     * Redistributions in binary form must reproduce the above copyright
  10 ;;       notice, this list of conditions and the following disclaimer in the
  11 ;;       documentation and/or other materials provided with the distribution.
  12 ;;     * Neither the name of Intel Corporation nor the names of its contributors
  13 ;;       may be used to endorse or promote products derived from this software
  14 ;;       without specific prior written permission.
  15 ;;
  16 ;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  17 ;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18 ;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  19 ;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
  20 ;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21 ;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
  22 ;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
  23 ;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  24 ;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  25 ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  26 ;;
  27
  28 %include "include/os.asm"
  29 %include "job_aes_hmac.asm"
  30 %include "mb_mgr_datastruct.asm"
  31 %include "include/reg_sizes.asm"
  32 %include "include/memcpy.asm"
  33 %include "include/const.inc"
  34
  35 extern sha512_x2_sse
  36
  37 section .data
  38 default rel
  39 align 16
  40 byteswap:       ;ddq 0x08090a0b0c0d0e0f0001020304050607
  41         dq 0x0001020304050607, 0x08090a0b0c0d0e0f
  42
  43 section .text
  44
  45 %ifndef FUNC
  46 %define FUNC submit_job_hmac_sha_512_sse
  47 %define SHA_X_DIGEST_SIZE 512
  48 %endif
  49
  50 %if 1
  51 %ifdef LINUX
  52 %define arg1    rdi
  53 %define arg2    rsi
  54 %define reg3    rcx
  55 %define reg4    rdx
  56 %else
  57 %define arg1    rcx
  58 %define arg2    rdx
  59 %define reg3    rdi
  60 %define reg4    rsi
  61 %endif
  62
  63 %define state   arg1
  64 %define job     arg2
  65 %define len2    arg2
  66
  67
  68 ; idx needs to be in rbx, rbp, r12-r15
  69 %define last_len        rbp
  70 %define idx             rbp
  71
  72 %define p               r11
  73 %define start_offset    r11
  74
  75 %define unused_lanes    rbx
  76 %define tmp4            rbx
  77
  78 %define job_rax         rax
  79 %define len             rax
  80
  81 %define size_offset     reg3
  82 %define tmp2            reg3
  83
  84 %define lane            reg4
  85 %define tmp3            reg4
  86
  87 %define extra_blocks    r8
  88
  89 %define tmp             r9
  90 %define p2              r9
  91
  92 %define lane_data       r10
  93
  94 %endif
  95
  96 ; This routine clobbers rbx, rbp, rsi, rdi
  97 struc STACK
  98 _gpr_save:      resq    4
  99 _rsp_save:      resq    1
 100 endstruc
 101
 102 ; JOB* FUNC(MB_MGR_HMAC_SHA_512_OOO *state, JOB_AES_HMAC *job)
 103 ; arg 1 : rcx : state
 104 ; arg 2 : rdx : job
 105 MKGLOBAL(FUNC,function,internal)
 106 FUNC:
 107
 108         mov     rax, rsp
 109         sub     rsp, STACK_size
 110         and     rsp, -16
 111
 112         mov     [rsp + _gpr_save + 8*0], rbx
 113         mov     [rsp + _gpr_save + 8*1], rbp
 114 %ifndef LINUX
 115         mov     [rsp + _gpr_save + 8*2], rsi
 116         mov     [rsp + _gpr_save + 8*3], rdi
 117 %endif
 118         mov     [rsp + _rsp_save], rax  ; original SP
 119
 120         mov     unused_lanes, [state + _unused_lanes_sha512]
 121         movzx   lane, BYTE(unused_lanes)
 122         shr     unused_lanes, 8
 123         imul    lane_data, lane, _SHA512_LANE_DATA_size
 124         lea     lane_data, [state + _ldata_sha512+ lane_data]
 125         mov     [state + _unused_lanes_sha512], unused_lanes
 126         mov     len, [job + _msg_len_to_hash_in_bytes]
 127         mov     tmp, len
 128         shr     tmp, 7  ; divide by 128, len in terms of sha512 blocks
 129
 130         mov     [lane_data + _job_in_lane_sha512], job
 131         mov     dword [lane_data + _outer_done_sha512], 0
 132
 133         movdqa  xmm0, [state + _lens_sha512]
 134         XPINSRW xmm0, xmm1, p, lane, tmp, scale_x16
 135         movdqa  [state + _lens_sha512], xmm0
 136
 137         mov     last_len, len
 138         and     last_len, 127
 139         lea     extra_blocks, [last_len + 17 + 127]
 140         shr     extra_blocks, 7
 141         mov     [lane_data + _extra_blocks_sha512], DWORD(extra_blocks)
 142
 143         mov     p, [job + _src]
 144         add     p, [job + _hash_start_src_offset_in_bytes]
 145         mov     [state + _args_data_ptr_sha512 + PTR_SZ*lane], p
 146
 147         cmp     len, 128
 148         jb      copy_lt128
 149
 150 fast_copy:
 151         add     p, len
 152 %assign I 0
 153 %rep 2
 154         movdqu  xmm0, [p - 128 + I*4*16 + 0*16]
 155         movdqu  xmm1, [p - 128 + I*4*16 + 1*16]
 156         movdqu  xmm2, [p - 128 + I*4*16 + 2*16]
 157         movdqu  xmm3, [p - 128 + I*4*16 + 3*16]
 158         movdqa  [lane_data + _extra_block_sha512 + I*4*16 + 0*16], xmm0
 159         movdqa  [lane_data + _extra_block_sha512 + I*4*16 + 1*16], xmm1
 160         movdqa  [lane_data + _extra_block_sha512 + I*4*16 + 2*16], xmm2
 161         movdqa  [lane_data + _extra_block_sha512 + I*4*16 + 3*16], xmm3
 162 %assign I (I+1)
 163 %endrep
 164 end_fast_copy:
 165
 166         mov     size_offset, extra_blocks
 167         shl     size_offset, 7
 168         sub     size_offset, last_len
 169         add     size_offset, 128-8
 170         mov     [lane_data + _size_offset_sha512], DWORD(size_offset)
 171         mov     start_offset, 128
 172         sub     start_offset, last_len
 173         mov     [lane_data + _start_offset_sha512], DWORD(start_offset)
 174
 175         lea     tmp, [8*128 + 8*len]
 176         bswap   tmp
 177         mov     [lane_data + _extra_block_sha512 + size_offset], tmp
 178
 179         mov     tmp, [job + _auth_key_xor_ipad]
 180  %assign I 0
 181  %rep 4
 182         movdqu  xmm0, [tmp + I * 2 * SHA512_DIGEST_WORD_SIZE]
 183         movq    [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*lane + (2*I)*SHA512_DIGEST_ROW_SIZE], xmm0
 184         pextrq  [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*lane + (2*I + 1)*SHA512_DIGEST_ROW_SIZE], xmm0, 1
 185  %assign I (I+1)
 186  %endrep
 187         test    len, ~127
 188         jnz     ge128_bytes
 189
 190 lt128_bytes:
 191         movdqa  xmm0, [state + _lens_sha512]
 192         XPINSRW xmm0, xmm1, tmp, lane, extra_blocks, scale_x16
 193         movdqa  [state + _lens_sha512], xmm0
 194
 195         lea     tmp, [lane_data + _extra_block_sha512 + start_offset]
 196         mov     [state + _args_data_ptr_sha512 + PTR_SZ*lane], tmp ;; 8 to hold a UINT8
 197         mov     dword [lane_data + _extra_blocks_sha512], 0
 198
 199 ge128_bytes:
 200         cmp     unused_lanes, 0xff
 201         jne     return_null
 202         jmp     start_loop
 203
 204         align   16
 205 start_loop:
 206         ; Find min length
 207         movdqa  xmm0, [state + _lens_sha512]
 208         phminposuw      xmm1, xmm0
 209         pextrw  DWORD(len2), xmm1, 0    ; min value
 210         pextrw  DWORD(idx), xmm1, 1     ; min index (0...1)
 211         cmp     len2, 0
 212         je      len_is_0
 213
 214         pshuflw xmm1, xmm1, 0XA0
 215         psubw   xmm0, xmm1
 216         movdqa  [state + _lens_sha512], xmm0
 217
 218         ; "state" and "args" are the same address, arg1
 219         ; len is arg2
 220         call    sha512_x2_sse
 221         ; state and idx are intact
 222
 223 len_is_0:
 224         ; process completed job "idx"
 225         imul    lane_data, idx, _SHA512_LANE_DATA_size
 226         lea     lane_data, [state + _ldata_sha512 + lane_data]
 227         mov     DWORD(extra_blocks), [lane_data + _extra_blocks_sha512]
 228         cmp     extra_blocks, 0
 229         jne     proc_extra_blocks
 230         cmp     dword [lane_data + _outer_done_sha512], 0
 231         jne     end_loop
 232
 233 proc_outer:
 234         mov     dword [lane_data + _outer_done_sha512], 1
 235         mov     DWORD(size_offset), [lane_data + _size_offset_sha512]
 236         mov     qword [lane_data + _extra_block_sha512 + size_offset], 0
 237
 238         movdqa  xmm0, [state + _lens_sha512]
 239         XPINSRW xmm0, xmm1, tmp, idx, 1, scale_x16
 240         movdqa  [state + _lens_sha512], xmm0
 241
 242         lea     tmp, [lane_data + _outer_block_sha512]
 243         mov     job, [lane_data + _job_in_lane_sha512]
 244         mov     [state + _args_data_ptr_sha512 + PTR_SZ*idx], tmp
 245
 246 %assign I 0
 247 %rep (SHA_X_DIGEST_SIZE / (8 * 16))
 248         movq    xmm0, [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + (2*I)*SHA512_DIGEST_ROW_SIZE]
 249         pinsrq  xmm0, [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + (2*I + 1)*SHA512_DIGEST_ROW_SIZE], 1
 250         pshufb  xmm0, [rel byteswap]
 251         movdqa  [lane_data + _outer_block_sha512 + I*16], xmm0
 252 %assign I (I+1)
 253 %endrep
 254
 255         mov     tmp, [job + _auth_key_xor_opad]
 256 %assign I 0
 257 %rep 4
 258         movdqu  xmm0, [tmp + I*16]
 259         movq    [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 2*I*SHA512_DIGEST_ROW_SIZE], xmm0
 260         pextrq  [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + (2*I + 1)*SHA512_DIGEST_ROW_SIZE], xmm0, 1
 261 %assign I (I+1)
 262 %endrep
 263         jmp     start_loop
 264
 265         align   16
 266 proc_extra_blocks:
 267         mov     DWORD(start_offset), [lane_data + _start_offset_sha512]
 268
 269         movdqa  xmm0, [state + _lens_sha512]
 270         XPINSRW xmm0, xmm1, tmp, idx, extra_blocks, scale_x16
 271         movdqa  [state + _lens_sha512], xmm0
 272
 273         lea     tmp, [lane_data + _extra_block_sha512 + start_offset]
 274         mov     [state + _args_data_ptr_sha512 + PTR_SZ*idx], tmp
 275         mov     dword [lane_data + _extra_blocks_sha512], 0
 276         jmp     start_loop
 277
 278         align   16
 279 copy_lt128:
 280         ;; less than one message block of data
 281         ;; beginning of source block
 282         ;; destination extra block but backwards by len from where 0x80 pre-populated
 283         lea     p2, [lane_data + _extra_block  + 128]
 284         sub     p2, len
 285         memcpy_sse_128_1 p2, p, len, tmp4, tmp2, xmm0, xmm1, xmm2, xmm3
 286         mov     unused_lanes, [state + _unused_lanes_sha512]
 287         jmp     end_fast_copy
 288
 289 return_null:
 290         xor     job_rax, job_rax
 291         jmp     return
 292
 293         align   16
 294 end_loop:
 295         mov     job_rax, [lane_data + _job_in_lane_sha512]
 296         mov     unused_lanes, [state + _unused_lanes_sha512]
 297         mov     qword [lane_data + _job_in_lane_sha512], 0
 298         or      dword [job_rax + _status], STS_COMPLETED_HMAC
 299         shl     unused_lanes, 8
 300         or      unused_lanes, idx
 301         mov     [state + _unused_lanes_sha512], unused_lanes
 302
 303         mov     p, [job_rax + _auth_tag_output]
 304
 305 %if (SHA_X_DIGEST_SIZE != 384)
 306         cmp     qword [job_rax + _auth_tag_output_len_in_bytes], 32
 307         jne     copy_full_digest
 308 %else
 309         cmp     qword [job_rax + _auth_tag_output_len_in_bytes], 24
 310         jne     copy_full_digest
 311 %endif
 312
 313         ;; copy 32 bytes for SHA512 / 24 bytes for SHA384
 314         mov     QWORD(tmp),  [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 0*SHA512_DIGEST_ROW_SIZE]
 315         mov     QWORD(tmp2), [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 1*SHA512_DIGEST_ROW_SIZE]
 316         mov     QWORD(tmp3), [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 2*SHA512_DIGEST_ROW_SIZE]
 317 %if (SHA_X_DIGEST_SIZE != 384)
 318         mov     QWORD(tmp4), [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 3*SHA512_DIGEST_ROW_SIZE] ; this line of code will run only for SHA512
 319 %endif
 320         bswap   QWORD(tmp)
 321         bswap   QWORD(tmp2)
 322         bswap   QWORD(tmp3)
 323 %if (SHA_X_DIGEST_SIZE != 384)
 324         bswap   QWORD(tmp4)
 325 %endif
 326         mov     [p + 0*8], QWORD(tmp)
 327         mov     [p + 1*8], QWORD(tmp2)
 328         mov     [p + 2*8], QWORD(tmp3)
 329 %if (SHA_X_DIGEST_SIZE != 384)
 330         mov     [p + 3*8], QWORD(tmp4)
 331 %endif
 332         jmp     clear_ret
 333
 334 copy_full_digest:
 335         ;; copy 64 bytes for SHA512 / 48 bytes for SHA384
 336         mov     QWORD(tmp),  [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 0*SHA512_DIGEST_ROW_SIZE]
 337         mov     QWORD(tmp2), [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 1*SHA512_DIGEST_ROW_SIZE]
 338         mov     QWORD(tmp3), [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 2*SHA512_DIGEST_ROW_SIZE]
 339         mov     QWORD(tmp4), [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 3*SHA512_DIGEST_ROW_SIZE] ; this line of code will run only for SHA512
 340         bswap   QWORD(tmp)
 341         bswap   QWORD(tmp2)
 342         bswap   QWORD(tmp3)
 343         bswap   QWORD(tmp4)
 344         mov     [p + 0*8], QWORD(tmp)
 345         mov     [p + 1*8], QWORD(tmp2)
 346         mov     [p + 2*8], QWORD(tmp3)
 347         mov     [p + 3*8], QWORD(tmp4)
 348         mov     QWORD(tmp),  [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 4*SHA512_DIGEST_ROW_SIZE]
 349         mov     QWORD(tmp2), [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 5*SHA512_DIGEST_ROW_SIZE]
 350 %if (SHA_X_DIGEST_SIZE != 384)
 351         mov     QWORD(tmp3), [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 6*SHA512_DIGEST_ROW_SIZE]
 352         mov     QWORD(tmp4), [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 7*SHA512_DIGEST_ROW_SIZE] ; this line of code will run only for SHA512
 353 %endif
 354         bswap   QWORD(tmp)
 355         bswap   QWORD(tmp2)
 356 %if (SHA_X_DIGEST_SIZE != 384)
 357         bswap   QWORD(tmp3)
 358         bswap   QWORD(tmp4)
 359 %endif
 360         mov     [p + 4*8], QWORD(tmp)
 361         mov     [p + 5*8], QWORD(tmp2)
 362 %if (SHA_X_DIGEST_SIZE != 384)
 363         mov     [p + 6*8], QWORD(tmp3)
 364         mov     [p + 7*8], QWORD(tmp4)
 365 %endif
 366
 367 clear_ret:
 368
 369 %ifdef SAFE_DATA
 370         ;; Clear digest (48B/64B), outer_block (48B/64B) and extra_block (128B) of returned job
 371 %assign J 0
 372 %rep 6
 373         mov     qword [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + J*SHA512_DIGEST_ROW_SIZE], 0
 374 %assign J (J+1)
 375 %endrep
 376 %if (SHA_X_DIGEST_SIZE != 384)
 377         mov     qword [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 6*SHA256_DIGEST_ROW_SIZE], 0
 378         mov     qword [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 7*SHA256_DIGEST_ROW_SIZE], 0
 379 %endif
 380
 381         pxor    xmm0, xmm0
 382         imul    lane_data, idx, _SHA512_LANE_DATA_size
 383         lea     lane_data, [state + _ldata_sha512 + lane_data]
 384         ;; Clear first 128 bytes of extra_block
 385 %assign offset 0
 386 %rep 8
 387         movdqa  [lane_data + _extra_block + offset], xmm0
 388 %assign offset (offset + 16)
 389 %endrep
 390
 391         ;; Clear first 48 bytes (SHA-384) or 64 bytes (SHA-512) of outer_block
 392         movdqa  [lane_data + _outer_block], xmm0
 393         movdqa  [lane_data + _outer_block + 16], xmm0
 394         movdqa  [lane_data + _outer_block + 32], xmm0
 395 %if (SHA_X_DIGEST_SIZE != 384)
 396         movdqa  [lane_data + _outer_block + 48], xmm0
 397 %endif
 398 %endif ;; SAFE_DATA
 399
 400 return:
 401         mov     rbx, [rsp + _gpr_save + 8*0]
 402         mov     rbp, [rsp + _gpr_save + 8*1]
 403 %ifndef LINUX
 404         mov     rsi, [rsp + _gpr_save + 8*2]
 405         mov     rdi, [rsp + _gpr_save + 8*3]
 406 %endif
 407         mov     rsp, [rsp + _rsp_save]  ; original SP
 408         ret
 409
 410 %ifdef LINUX
 411 section .note.GNU-stack noalloc noexec nowrite progbits
 412 %endif