ceph/src/spdk/intel-ipsec-mb/avx/mb_mgr_hmac_sha_256_submit_avx.asm

   1 ;;
   2 ;; Copyright (c) 2012-2018, Intel Corporation
   3 ;;
   4 ;; Redistribution and use in source and binary forms, with or without
   5 ;; modification, are permitted provided that the following conditions are met:
   6 ;;
   7 ;;     * Redistributions of source code must retain the above copyright notice,
   8 ;;       this list of conditions and the following disclaimer.
   9 ;;     * Redistributions in binary form must reproduce the above copyright
  10 ;;       notice, this list of conditions and the following disclaimer in the
  11 ;;       documentation and/or other materials provided with the distribution.
  12 ;;     * Neither the name of Intel Corporation nor the names of its contributors
  13 ;;       may be used to endorse or promote products derived from this software
  14 ;;       without specific prior written permission.
  15 ;;
  16 ;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  17 ;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18 ;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  19 ;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
  20 ;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21 ;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
  22 ;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
  23 ;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  24 ;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  25 ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  26 ;;
  27
  28 %include "include/os.asm"
  29 %include "job_aes_hmac.asm"
  30 %include "mb_mgr_datastruct.asm"
  31 %include "include/reg_sizes.asm"
  32 %include "include/memcpy.asm"
  33 %include "include/const.inc"
  34
  35 extern sha_256_mult_avx
  36
  37 section .data
  38 default rel
  39 align 16
  40 byteswap:       ;ddq 0x0c0d0e0f08090a0b0405060700010203
  41         dq 0x0405060700010203, 0x0c0d0e0f08090a0b
  42
  43 section .text
  44
  45 %ifndef FUNC
  46 %define FUNC submit_job_hmac_sha_256_avx
  47 %endif
  48
  49 %if 1
  50 %ifdef LINUX
  51 %define arg1    rdi
  52 %define arg2    rsi
  53 %define reg3    rcx
  54 %define reg4    rdx
  55 %else
  56 %define arg1    rcx
  57 %define arg2    rdx
  58 %define reg3    rdi
  59 %define reg4    rsi
  60 %endif
  61
  62 %define state   arg1
  63 %define job     arg2
  64 %define len2    arg2
  65
  66
  67 ; idx needs to be in rbx, rbp, r13-r15
  68 %define last_len        rbp
  69 %define idx             rbp
  70
  71 %define p               r11
  72 %define start_offset    r11
  73
  74 %define unused_lanes    rbx
  75 %define tmp4            rbx
  76
  77 %define job_rax         rax
  78 %define len             rax
  79
  80 %define size_offset     reg3
  81 %define tmp2            reg3
  82
  83 %define lane            reg4
  84 %define tmp3            reg4
  85
  86 %define extra_blocks    r8
  87
  88 %define tmp             r9
  89 %define p2              r9
  90
  91 %define lane_data       r10
  92
  93 %endif
  94
  95 ; This routine clobbers rbx, rbp, rsi, rdi; called routine also clobbers r12
  96 struc STACK
  97 _gpr_save:      resq    5
  98 _rsp_save:      resq    1
  99 endstruc
 100
 101 ; JOB* FUNC(MB_MGR_HMAC_SHA_256_OOO *state, JOB_AES_HMAC *job)
 102 ; arg 1 : rcx : state
 103 ; arg 2 : rdx : job
 104 MKGLOBAL(FUNC,function,internal)
 105 FUNC:
 106
 107         mov     rax, rsp
 108         sub     rsp, STACK_size
 109         and     rsp, -16
 110
 111         mov     [rsp + _gpr_save + 8*0], rbx
 112         mov     [rsp + _gpr_save + 8*1], rbp
 113         mov     [rsp + _gpr_save + 8*2], r12
 114 %ifndef LINUX
 115         mov     [rsp + _gpr_save + 8*3], rsi
 116         mov     [rsp + _gpr_save + 8*4], rdi
 117 %endif
 118         mov     [rsp + _rsp_save], rax  ; original SP
 119
 120         mov     unused_lanes, [state + _unused_lanes_sha256]
 121         movzx   lane, BYTE(unused_lanes)
 122         shr     unused_lanes, 8
 123         imul    lane_data, lane, _HMAC_SHA1_LANE_DATA_size
 124         lea     lane_data, [state + _ldata_sha256 + lane_data]
 125         mov     [state + _unused_lanes_sha256], unused_lanes
 126         mov     len, [job + _msg_len_to_hash_in_bytes]
 127         mov     tmp, len
 128         shr     tmp, 6  ; divide by 64, len in terms of blocks
 129
 130         mov     [lane_data + _job_in_lane], job
 131         mov     dword [lane_data + _outer_done], 0
 132
 133         vmovdqa xmm0, [state + _lens_sha256]
 134         XVPINSRW xmm0, xmm1, p, lane, tmp, scale_x16
 135         vmovdqa [state + _lens_sha256], xmm0
 136
 137         mov     last_len, len
 138         and     last_len, 63
 139         lea     extra_blocks, [last_len + 9 + 63]
 140         shr     extra_blocks, 6
 141         mov     [lane_data + _extra_blocks], DWORD(extra_blocks)
 142
 143         mov     p, [job + _src]
 144         add     p, [job + _hash_start_src_offset_in_bytes]
 145         mov     [state + _args_data_ptr_sha256 + 8*lane], p
 146
 147         cmp     len, 64
 148         jb      copy_lt64
 149
 150 fast_copy:
 151         add             p, len
 152         vmovdqu xmm0, [p - 64 + 0*16]
 153         vmovdqu xmm1, [p - 64 + 1*16]
 154         vmovdqu xmm2, [p - 64 + 2*16]
 155         vmovdqu xmm3, [p - 64 + 3*16]
 156         vmovdqa [lane_data + _extra_block + 0*16], xmm0
 157         vmovdqa [lane_data + _extra_block + 1*16], xmm1
 158         vmovdqa [lane_data + _extra_block + 2*16], xmm2
 159         vmovdqa [lane_data + _extra_block + 3*16], xmm3
 160 end_fast_copy:
 161
 162         mov     size_offset, extra_blocks
 163         shl     size_offset, 6
 164         sub     size_offset, last_len
 165         add     size_offset, 64-8
 166         mov     [lane_data + _size_offset], DWORD(size_offset)
 167         mov     start_offset, 64
 168         sub     start_offset, last_len
 169         mov     [lane_data + _start_offset], DWORD(start_offset)
 170
 171         lea     tmp, [8*64 + 8*len]
 172         bswap   tmp
 173         mov     [lane_data + _extra_block + size_offset], tmp
 174
 175         mov     tmp, [job + _auth_key_xor_ipad]
 176         vmovdqu xmm0, [tmp]
 177         vmovdqu xmm1,  [tmp + 4*4]
 178         vmovd   [state + _args_digest_sha256 + 4*lane + 0*SHA256_DIGEST_ROW_SIZE], xmm0
 179         vpextrd [state + _args_digest_sha256 + 4*lane + 1*SHA256_DIGEST_ROW_SIZE], xmm0, 1
 180         vpextrd [state + _args_digest_sha256 + 4*lane + 2*SHA256_DIGEST_ROW_SIZE], xmm0, 2
 181         vpextrd [state + _args_digest_sha256 + 4*lane + 3*SHA256_DIGEST_ROW_SIZE], xmm0, 3
 182         vmovd   [state + _args_digest_sha256 + 4*lane + 4*SHA256_DIGEST_ROW_SIZE], xmm1
 183         vpextrd [state + _args_digest_sha256 + 4*lane + 5*SHA256_DIGEST_ROW_SIZE], xmm1, 1
 184         vpextrd [state + _args_digest_sha256 + 4*lane + 6*SHA256_DIGEST_ROW_SIZE], xmm1, 2
 185         vpextrd [state + _args_digest_sha256 + 4*lane + 7*SHA256_DIGEST_ROW_SIZE], xmm1, 3
 186
 187         test    len, ~63
 188         jnz     ge64_bytes
 189
 190 lt64_bytes:
 191         vmovdqa xmm0, [state + _lens_sha256]
 192         XVPINSRW xmm0, xmm1, tmp, lane, extra_blocks, scale_x16
 193         vmovdqa [state + _lens_sha256], xmm0
 194
 195         lea     tmp, [lane_data + _extra_block + start_offset]
 196         mov     [state + _args_data_ptr_sha256 + 8*lane], tmp
 197         mov     dword [lane_data + _extra_blocks], 0
 198
 199 ge64_bytes:
 200         cmp     unused_lanes, 0xff
 201         jne     return_null
 202         jmp     start_loop
 203
 204         align   16
 205 start_loop:
 206         ; Find min length
 207         vmovdqa xmm0, [state + _lens_sha256]
 208         vphminposuw     xmm1, xmm0
 209         vpextrw DWORD(len2), xmm1, 0    ; min value
 210         vpextrw DWORD(idx), xmm1, 1     ; min index (0...3)
 211         cmp     len2, 0
 212         je      len_is_0
 213
 214         vpshuflw xmm1, xmm1, 0
 215         vpsubw  xmm0, xmm0, xmm1
 216         vmovdqa [state + _lens_sha256], xmm0
 217
 218         ; "state" and "args" are the same address, arg1
 219         ; len is arg2
 220         call    sha_256_mult_avx
 221         ; state and idx are intact
 222
 223 len_is_0:
 224         ; process completed job "idx"
 225         imul    lane_data, idx, _HMAC_SHA1_LANE_DATA_size
 226         lea     lane_data, [state + _ldata_sha256 + lane_data]
 227         mov     DWORD(extra_blocks), [lane_data + _extra_blocks]
 228         cmp     extra_blocks, 0
 229         jne     proc_extra_blocks
 230         cmp     dword [lane_data + _outer_done], 0
 231         jne     end_loop
 232
 233 proc_outer:
 234         mov     dword [lane_data + _outer_done], 1
 235         mov     DWORD(size_offset), [lane_data + _size_offset]
 236         mov     qword [lane_data + _extra_block + size_offset], 0
 237
 238         vmovdqa xmm0, [state + _lens_sha256]
 239         XVPINSRW xmm0, xmm1, tmp, idx, 1, scale_x16
 240         vmovdqa [state + _lens_sha256], xmm0
 241
 242         lea     tmp, [lane_data + _outer_block]
 243         mov     job, [lane_data + _job_in_lane]
 244         mov     [state + _args_data_ptr_sha256 + 8*idx], tmp
 245
 246         vmovd   xmm0, [state + _args_digest_sha256 + 4*idx + 0*SHA256_DIGEST_ROW_SIZE]
 247         vpinsrd xmm0, xmm0, [state + _args_digest_sha256 + 4*idx + 1*SHA256_DIGEST_ROW_SIZE], 1
 248         vpinsrd xmm0, xmm0, [state + _args_digest_sha256 + 4*idx + 2*SHA256_DIGEST_ROW_SIZE], 2
 249         vpinsrd xmm0, xmm0, [state + _args_digest_sha256 + 4*idx + 3*SHA256_DIGEST_ROW_SIZE], 3
 250         vpshufb xmm0, xmm0, [rel byteswap]
 251         vmovd   xmm1, [state + _args_digest_sha256 + 4*idx + 4*SHA256_DIGEST_ROW_SIZE]
 252         vpinsrd xmm1, xmm1, [state + _args_digest_sha256 + 4*idx + 5*SHA256_DIGEST_ROW_SIZE], 1
 253         vpinsrd xmm1, xmm1, [state + _args_digest_sha256 + 4*idx + 6*SHA256_DIGEST_ROW_SIZE], 2
 254 %ifndef SHA224
 255         vpinsrd xmm1, xmm1, [state + _args_digest_sha256 + 4*idx + 7*SHA256_DIGEST_ROW_SIZE], 3
 256 %endif
 257         vpshufb xmm1, xmm1, [rel byteswap]
 258         vmovdqa [lane_data + _outer_block], xmm0
 259         vmovdqa [lane_data + _outer_block + 4*4], xmm1
 260 %ifdef SHA224
 261         mov             dword [lane_data + _outer_block + 7*4], 0x80
 262 %endif
 263
 264         mov     tmp, [job + _auth_key_xor_opad]
 265         vmovdqu xmm0, [tmp]
 266         vmovdqu xmm1, [tmp + 4*4]
 267         vmovd   [state + _args_digest_sha256 + 4*idx + 0*SHA256_DIGEST_ROW_SIZE], xmm0
 268         vpextrd [state + _args_digest_sha256 + 4*idx + 1*SHA256_DIGEST_ROW_SIZE], xmm0, 1
 269         vpextrd [state + _args_digest_sha256 + 4*idx + 2*SHA256_DIGEST_ROW_SIZE], xmm0, 2
 270         vpextrd [state + _args_digest_sha256 + 4*idx + 3*SHA256_DIGEST_ROW_SIZE], xmm0, 3
 271         vmovd   [state + _args_digest_sha256 + 4*idx + 4*SHA256_DIGEST_ROW_SIZE], xmm1
 272         vpextrd [state + _args_digest_sha256 + 4*idx + 5*SHA256_DIGEST_ROW_SIZE], xmm1, 1
 273         vpextrd [state + _args_digest_sha256 + 4*idx + 6*SHA256_DIGEST_ROW_SIZE], xmm1, 2
 274         vpextrd [state + _args_digest_sha256 + 4*idx + 7*SHA256_DIGEST_ROW_SIZE], xmm1, 3
 275
 276         jmp     start_loop
 277
 278         align   16
 279 proc_extra_blocks:
 280         mov     DWORD(start_offset), [lane_data + _start_offset]
 281
 282         vmovdqa xmm0, [state + _lens_sha256]
 283         XVPINSRW xmm0, xmm1, tmp, idx, extra_blocks, scale_x16
 284         vmovdqa [state + _lens_sha256], xmm0
 285
 286         lea     tmp, [lane_data + _extra_block + start_offset]
 287         mov     [state + _args_data_ptr_sha256 + 8*idx], tmp
 288         mov     dword [lane_data + _extra_blocks], 0
 289         jmp     start_loop
 290
 291         align   16
 292
 293 copy_lt64:
 294         ;; less than one message block of data
 295         ;; beginning of source block
 296         ;; destination extrablock but backwards by len from where 0x80 pre-populated
 297         ;; p2 clobbers unused_lanes, undo before exit
 298         lea     p2, [lane_data + _extra_block  + 64]
 299         sub     p2, len
 300         memcpy_avx_64_1 p2, p, len, tmp4, tmp2, xmm0, xmm1, xmm2, xmm3
 301         mov     unused_lanes, [state + _unused_lanes_sha256]
 302         jmp     end_fast_copy
 303
 304 return_null:
 305         xor     job_rax, job_rax
 306         jmp     return
 307
 308         align   16
 309 end_loop:
 310         mov     job_rax, [lane_data + _job_in_lane]
 311         mov     unused_lanes, [state + _unused_lanes_sha256]
 312         mov     qword [lane_data + _job_in_lane], 0
 313         or      dword [job_rax + _status], STS_COMPLETED_HMAC
 314         shl     unused_lanes, 8
 315         or      unused_lanes, idx
 316         mov     [state + _unused_lanes_sha256], unused_lanes
 317
 318         mov     p, [job_rax + _auth_tag_output]
 319
 320 %ifdef SHA224
 321         cmp     qword [job_rax + _auth_tag_output_len_in_bytes], 14
 322         jne     copy_full_digest
 323 %else
 324         cmp     qword [job_rax + _auth_tag_output_len_in_bytes], 16
 325         jne     copy_full_digest
 326 %endif
 327         ; copy 14 bytes for SHA224 / 16 bytes for SHA256
 328         mov     DWORD(tmp),  [state + _args_digest_sha256 + 4*idx + 0*SHA256_DIGEST_ROW_SIZE]
 329         mov     DWORD(tmp2), [state + _args_digest_sha256 + 4*idx + 1*SHA256_DIGEST_ROW_SIZE]
 330         mov     DWORD(tmp3), [state + _args_digest_sha256 + 4*idx + 2*SHA256_DIGEST_ROW_SIZE]
 331         mov     DWORD(tmp4), [state + _args_digest_sha256 + 4*idx + 3*SHA256_DIGEST_ROW_SIZE]
 332         bswap   DWORD(tmp)
 333         bswap   DWORD(tmp2)
 334         bswap   DWORD(tmp3)
 335         bswap   DWORD(tmp4)
 336         mov     [p + 0*4], DWORD(tmp)
 337         mov     [p + 1*4], DWORD(tmp2)
 338         mov     [p + 2*4], DWORD(tmp3)
 339 %ifdef SHA224
 340         mov     [p + 3*4], WORD(tmp4)
 341 %else
 342         mov     [p + 3*4], DWORD(tmp4)
 343 %endif
 344         jmp     clear_ret
 345
 346 copy_full_digest:
 347         ;; copy 28 bytes for SHA224 / 32 bytes for SHA256
 348         mov     DWORD(tmp),  [state + _args_digest_sha256 + 4*idx + 0*SHA256_DIGEST_ROW_SIZE]
 349         mov     DWORD(tmp2), [state + _args_digest_sha256 + 4*idx + 1*SHA256_DIGEST_ROW_SIZE]
 350         mov     DWORD(tmp3), [state + _args_digest_sha256 + 4*idx + 2*SHA256_DIGEST_ROW_SIZE]
 351         mov     DWORD(tmp4), [state + _args_digest_sha256 + 4*idx + 3*SHA256_DIGEST_ROW_SIZE]
 352         bswap   DWORD(tmp)
 353         bswap   DWORD(tmp2)
 354         bswap   DWORD(tmp3)
 355         bswap   DWORD(tmp4)
 356         mov     [p + 0*4], DWORD(tmp)
 357         mov     [p + 1*4], DWORD(tmp2)
 358         mov     [p + 2*4], DWORD(tmp3)
 359         mov     [p + 3*4], DWORD(tmp4)
 360
 361         mov     DWORD(tmp),  [state + _args_digest_sha256 + 4*idx + 4*SHA256_DIGEST_ROW_SIZE]
 362         mov     DWORD(tmp2), [state + _args_digest_sha256 + 4*idx + 5*SHA256_DIGEST_ROW_SIZE]
 363         mov     DWORD(tmp3), [state + _args_digest_sha256 + 4*idx + 6*SHA256_DIGEST_ROW_SIZE]
 364 %ifndef SHA224
 365         mov     DWORD(tmp4), [state + _args_digest_sha256 + 4*idx + 7*SHA256_DIGEST_ROW_SIZE]
 366 %endif
 367         bswap   DWORD(tmp)
 368         bswap   DWORD(tmp2)
 369         bswap   DWORD(tmp3)
 370 %ifndef SHA224
 371         bswap   DWORD(tmp4)
 372 %endif
 373         mov     [p + 4*4], DWORD(tmp)
 374         mov     [p + 5*4], DWORD(tmp2)
 375         mov     [p + 6*4], DWORD(tmp3)
 376 %ifndef SHA224
 377         mov     [p + 7*4], DWORD(tmp4)
 378 %endif
 379
 380 clear_ret:
 381
 382 %ifdef SAFE_DATA
 383         ;; Clear digest (28B/32B), outer_block (28B/32B) and extra_block (64B) of returned job
 384 %assign J 0
 385 %rep 7
 386         mov     dword [state + _args_digest_sha256 + SHA256_DIGEST_WORD_SIZE*idx + J*SHA256_DIGEST_ROW_SIZE], 0
 387 %assign J (J+1)
 388 %endrep
 389 %ifndef SHA224
 390         mov     dword [state + _args_digest_sha256 + SHA256_DIGEST_WORD_SIZE*idx + 7*SHA256_DIGEST_ROW_SIZE], 0
 391 %endif
 392
 393         vpxor   xmm0, xmm0
 394         imul    lane_data, idx, _HMAC_SHA1_LANE_DATA_size
 395         lea     lane_data, [state + _ldata_sha256 + lane_data]
 396         ;; Clear first 64 bytes of extra_block
 397 %assign offset 0
 398 %rep 4
 399         vmovdqa [lane_data + _extra_block + offset], xmm0
 400 %assign offset (offset + 16)
 401 %endrep
 402
 403         ;; Clear first 28 bytes (SHA-224) or 32 bytes (SHA-256) of outer_block
 404         vmovdqa [lane_data + _outer_block], xmm0
 405 %ifdef SHA224
 406         mov     qword [lane_data + _outer_block + 16], 0
 407         mov     dword [lane_data + _outer_block + 24], 0
 408 %else
 409         vmovdqa [lane_data + _outer_block + 16], xmm0
 410 %endif
 411 %endif ;; SAFE_DATA
 412
 413 return:
 414
 415         mov     rbx, [rsp + _gpr_save + 8*0]
 416         mov     rbp, [rsp + _gpr_save + 8*1]
 417         mov     r12, [rsp + _gpr_save + 8*2]
 418 %ifndef LINUX
 419         mov     rsi, [rsp + _gpr_save + 8*3]
 420         mov     rdi, [rsp + _gpr_save + 8*4]
 421 %endif
 422         mov     rsp, [rsp + _rsp_save]  ; original SP
 423
 424         ret
 425
 426 %ifdef LINUX
 427 section .note.GNU-stack noalloc noexec nowrite progbits
 428 %endif