ceph/src/spdk/intel-ipsec-mb/avx512/mb_mgr_hmac_sha_512_submit_avx512.asm

   1 ;;
   2 ;; Copyright (c) 2017-2018, Intel Corporation
   3 ;;
   4 ;; Redistribution and use in source and binary forms, with or without
   5 ;; modification, are permitted provided that the following conditions are met:
   6 ;;
   7 ;;     * Redistributions of source code must retain the above copyright notice,
   8 ;;       this list of conditions and the following disclaimer.
   9 ;;     * Redistributions in binary form must reproduce the above copyright
  10 ;;       notice, this list of conditions and the following disclaimer in the
  11 ;;       documentation and/or other materials provided with the distribution.
  12 ;;     * Neither the name of Intel Corporation nor the names of its contributors
  13 ;;       may be used to endorse or promote products derived from this software
  14 ;;       without specific prior written permission.
  15 ;;
  16 ;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  17 ;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18 ;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  19 ;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
  20 ;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21 ;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
  22 ;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
  23 ;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  24 ;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  25 ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  26 ;;
  27
  28 ;; In System V AMD64 ABI
  29 ;;      calle saves: RBX, RBP, R12-R15
  30 ;; Windows x64 ABI
  31 ;;      calle saves: RBX, RBP, RDI, RSI, RSP, R12-R15
  32 ;;
  33 ;; Clobbers ZMM0-31
  34
  35 %include "os.asm"
  36 %include "job_aes_hmac.asm"
  37 %include "mb_mgr_datastruct.asm"
  38 %include "reg_sizes.asm"
  39 %include "memcpy.asm"
  40
  41 extern sha512_x8_avx512
  42
  43 section .data
  44 default rel
  45
  46 align 16
  47 dupw:   ;ddq 0x01000100010001000100010001000100
  48         dq 0x0100010001000100, 0x0100010001000100
  49
  50 align 16
  51 byteswap:       ;ddq 0x08090a0b0c0d0e0f0001020304050607
  52         dq 0x0001020304050607, 0x08090a0b0c0d0e0f
  53
  54 section .text
  55
  56 %ifdef LINUX
  57 %define arg1    rdi
  58 %define arg2    rsi
  59 %define arg3    rcx
  60 %define arg4    rdx
  61 %else
  62 %define arg1    rcx
  63 %define arg2    rdx
  64 %define arg3    rdi
  65 %define arg4    rsi
  66 %endif
  67
  68 %define state   arg1
  69 %define job     arg2
  70 %define len2    arg2
  71
  72
  73 ; idx needs to be in rbp, r13, r14, r16
  74 %define last_len        rbp
  75 %define idx             rbp
  76
  77 %define p               r11
  78 %define start_offset    r11
  79
  80 %define unused_lanes    rbx
  81 %define tmp4            rbx
  82
  83 %define job_rax         rax
  84 %define len             rax
  85
  86 %define size_offset     arg3
  87 %define tmp2            arg3
  88
  89 %define lane            arg4
  90 %define tmp3            arg4
  91
  92 %define extra_blocks    r8
  93
  94 %define tmp             r9
  95 %define p2              r9
  96
  97 %define lane_data       r10
  98
  99 ; Define stack usage
 100
 101 ; we clobber rbx, rsi, rdi, rbp; called routine also clobbers r12
 102 struc STACK
 103 _gpr_save:      resq    8
 104 _rsp_save:      resq    1
 105 endstruc
 106
 107 ; JOB* FUNC(MB_MGR_HMAC_sha_512_OOO *state, JOB_AES_HMAC *job)
 108 ; arg 1 : rcx : state
 109 ; arg 2 : rdx : job
 110 align 64
 111 %ifndef SHA384
 112 MKGLOBAL(submit_job_hmac_sha_512_avx512,function,internal)
 113 %define SHA_X_DIGEST_SIZE 512
 114 submit_job_hmac_sha_512_avx512:
 115 %else
 116 MKGLOBAL(submit_job_hmac_sha_384_avx512,function,internal)
 117 %define SHA_X_DIGEST_SIZE 384
 118 submit_job_hmac_sha_384_avx512:
 119 %endif
 120
 121         mov     rax, rsp
 122         sub     rsp, STACK_size
 123         and     rsp, -32
 124         mov     [rsp + _gpr_save + 8*0], rbx
 125         mov     [rsp + _gpr_save + 8*1], rbp
 126         mov     [rsp + _gpr_save + 8*2], r12
 127         mov     [rsp + _gpr_save + 8*3], r13
 128         mov     [rsp + _gpr_save + 8*4], r14
 129         mov     [rsp + _gpr_save + 8*5], r15
 130 %ifndef LINUX
 131         mov     [rsp + _gpr_save + 8*6], rsi
 132         mov     [rsp + _gpr_save + 8*7], rdi
 133 %endif
 134         mov     [rsp + _rsp_save], rax  ; original SP
 135
 136         mov     unused_lanes, [state + _unused_lanes_sha512]
 137         mov     lane, unused_lanes
 138         and     lane, 15
 139         shr     unused_lanes, 4
 140         imul    lane_data, lane, _SHA512_LANE_DATA_size
 141         lea     lane_data, [state + _ldata_sha512 + lane_data]
 142         mov     [state + _unused_lanes_sha512], unused_lanes
 143         mov     len, [job + _msg_len_to_hash_in_bytes]
 144         mov     tmp, len
 145         shr     tmp, 7  ; divide by 128, len in terms of blocks
 146
 147         mov     [lane_data + _job_in_lane_sha512], job
 148         mov     dword [lane_data + _outer_done_sha512], 0
 149         mov     [state + _lens_sha512 + 2*lane], WORD(tmp) ; 2 is word size in bytes
 150
 151         mov     last_len, len
 152         and     last_len, 127
 153         lea     extra_blocks, [last_len + 17 + 127]
 154         shr     extra_blocks, 7
 155         mov     [lane_data + _extra_blocks_sha512], DWORD(extra_blocks)
 156
 157         mov     p, [job + _src]
 158         add     p, [job + _hash_start_src_offset_in_bytes]
 159         mov     [state + _args_data_ptr_sha512 + PTR_SZ*lane], p
 160
 161         cmp     len, 128
 162         jb      copy_lt128
 163
 164 fast_copy:
 165         add     p, len
 166         vmovdqu32       zmm0, [p - 128 + 0*64]
 167         vmovdqu32       zmm1, [p - 128 + 1*64]
 168         vmovdqu32       [lane_data + _extra_block_sha512 + 0*64], zmm0
 169         vmovdqu32       [lane_data + _extra_block_sha512 + 1*64], zmm1
 170 end_fast_copy:
 171
 172         mov     size_offset, extra_blocks
 173         shl     size_offset, 7
 174         sub     size_offset, last_len
 175         add     size_offset, 128-8
 176         mov     [lane_data + _size_offset_sha512], DWORD(size_offset)
 177         mov     start_offset, 128
 178         sub     start_offset, last_len
 179         mov     [lane_data + _start_offset_sha512], DWORD(start_offset)
 180
 181         lea     tmp, [8*128 + 8*len]
 182         bswap   tmp
 183         mov     [lane_data + _extra_block_sha512 + size_offset], tmp
 184
 185         mov     tmp, [job + _auth_key_xor_ipad]
 186
 187 %assign I 0
 188 %rep 4
 189      vmovdqu    xmm0, [tmp + I * 2 * SHA512_DIGEST_WORD_SIZE]
 190      vmovq      [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*lane + (2*I + 0)*SHA512_DIGEST_ROW_SIZE], xmm0
 191      vpextrq    [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*lane + (2*I + 1)*SHA512_DIGEST_ROW_SIZE], xmm0, 1
 192 %assign I (I+1)
 193 %endrep
 194
 195         test    len, ~127
 196         jnz     ge128_bytes
 197
 198 lt128_bytes:
 199         mov     [state + _lens_sha512 + 2*lane], WORD(extra_blocks)
 200         lea     tmp, [lane_data + _extra_block_sha512 + start_offset]
 201         mov     [state + _args_data_ptr_sha512 + PTR_SZ*lane], tmp ;; 8 to hold a UINT8
 202         mov     dword [lane_data + _extra_blocks_sha512], 0
 203
 204 ge128_bytes:
 205         cmp     unused_lanes, 0xf
 206         jne     return_null
 207         jmp     start_loop
 208
 209         align   32
 210 start_loop:
 211         ; Find min length
 212         vmovdqa xmm0, [state + _lens_sha512]
 213         vphminposuw     xmm1, xmm0
 214         vpextrw DWORD(len2), xmm1, 0    ; min value
 215         vpextrw DWORD(idx), xmm1, 1     ; min index (0...7)
 216         cmp     len2, 0
 217         je      len_is_0
 218
 219         vpshufb xmm1, [rel dupw]        ; duplicate words across all 8 lanes
 220         vpsubw  xmm0, xmm0, xmm1
 221         vmovdqa [state + _lens_sha512], xmm0
 222
 223         ; "state" and "args" are the same address, arg1
 224         ; len is arg2
 225         call    sha512_x8_avx512
 226         ; state and idx are intact
 227
 228 len_is_0:
 229         ; process completed job "idx"
 230         imul    lane_data, idx, _SHA512_LANE_DATA_size
 231         lea     lane_data, [state + _ldata_sha512 + lane_data]
 232         mov     DWORD(extra_blocks), [lane_data + _extra_blocks_sha512]
 233         cmp     extra_blocks, 0
 234         jne     proc_extra_blocks
 235         cmp     dword [lane_data + _outer_done_sha512], 0
 236         jne     end_loop
 237
 238 proc_outer:
 239         mov     dword [lane_data + _outer_done_sha512], 1
 240         mov     DWORD(size_offset), [lane_data + _size_offset_sha512]
 241         mov     qword [lane_data + _extra_block_sha512 + size_offset], 0
 242         mov     word [state + _lens_sha512 + 2*idx], 1
 243         lea     tmp, [lane_data + _outer_block_sha512]
 244         mov     job, [lane_data + _job_in_lane_sha512]
 245         mov     [state + _args_data_ptr_sha512 + PTR_SZ*idx], tmp
 246
 247 %assign I 0
 248 %rep (SHA_X_DIGEST_SIZE / (8 * 16))
 249         vmovq   xmm0, [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + (2*I + 0)*SHA512_DIGEST_ROW_SIZE]
 250         vpinsrq xmm0, [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + (2*I + 1)*SHA512_DIGEST_ROW_SIZE], 1
 251         vpshufb xmm0, [rel byteswap]
 252         vmovdqa [lane_data + _outer_block_sha512 + I * 2 * SHA512_DIGEST_WORD_SIZE], xmm0
 253 %assign I (I+1)
 254 %endrep
 255
 256         mov     tmp, [job + _auth_key_xor_opad]
 257 %assign I 0
 258 %rep 4
 259         vmovdqu xmm0, [tmp + I * 16]
 260         vmovq   [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + (2*I+0)*SHA512_DIGEST_ROW_SIZE], xmm0
 261         vpextrq [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + (2*I + 1)*SHA512_DIGEST_ROW_SIZE], xmm0, 1
 262 %assign I (I+1)
 263 %endrep
 264
 265         jmp     start_loop
 266
 267         align   32
 268 proc_extra_blocks:
 269         mov     DWORD(start_offset), [lane_data + _start_offset_sha512]
 270         mov     [state + _lens_sha512 + 2*idx], WORD(extra_blocks)
 271         lea     tmp, [lane_data + _extra_block_sha512 + start_offset]
 272         mov     [state + _args_data_ptr_sha512 + PTR_SZ*idx], tmp ;;  idx is index of shortest length message
 273         mov     dword [lane_data + _extra_blocks_sha512], 0
 274         jmp     start_loop
 275
 276         align   32
 277 copy_lt128:
 278         ;; less than one message block of data
 279         ;; destination extra block but backwards by len from where 0x80 pre-populated
 280         lea     p2, [lane_data + _extra_block  + 128]
 281         sub     p2, len
 282         memcpy_avx2_128_1 p2, p, len, tmp4, tmp2, ymm0, ymm1, ymm2, ymm3
 283         mov     unused_lanes, [state + _unused_lanes_sha512]
 284         jmp     end_fast_copy
 285
 286 return_null:
 287         xor     job_rax, job_rax
 288         jmp     return
 289
 290         align   32
 291 end_loop:
 292         mov     job_rax, [lane_data + _job_in_lane_sha512]
 293         mov     unused_lanes, [state + _unused_lanes_sha512]
 294         mov     qword [lane_data + _job_in_lane_sha512], 0
 295         or      dword [job_rax + _status], STS_COMPLETED_HMAC
 296         shl     unused_lanes, 4
 297         or      unused_lanes, idx
 298         mov     [state + _unused_lanes_sha512], unused_lanes
 299
 300         mov     p, [job_rax + _auth_tag_output]
 301
 302         ; below is the code for both SHA512 & SHA384. SHA512=32 bytes and SHA384=24 bytes
 303         mov     QWORD(tmp),  [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 0*SHA512_DIGEST_ROW_SIZE]
 304         mov     QWORD(tmp2), [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 1*SHA512_DIGEST_ROW_SIZE]
 305         mov     QWORD(tmp3), [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 2*SHA512_DIGEST_ROW_SIZE]
 306 %if (SHA_X_DIGEST_SIZE != 384)
 307         mov     QWORD(tmp4), [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 3*SHA512_DIGEST_ROW_SIZE]
 308 %endif
 309
 310         bswap   QWORD(tmp)
 311         bswap   QWORD(tmp2)
 312         bswap   QWORD(tmp3)
 313 %if (SHA_X_DIGEST_SIZE != 384)
 314         bswap   QWORD(tmp4)
 315 %endif
 316
 317         mov     [p + 0*8], QWORD(tmp)
 318         mov     [p + 1*8], QWORD(tmp2)
 319         mov     [p + 2*8], QWORD(tmp3)
 320 %if (SHA_X_DIGEST_SIZE != 384)
 321         mov     [p + 3*8], QWORD(tmp4)
 322 %endif
 323         vzeroupper
 324 return:
 325         mov     rbx, [rsp + _gpr_save + 8*0]
 326         mov     rbp, [rsp + _gpr_save + 8*1]
 327         mov     r12, [rsp + _gpr_save + 8*2]
 328         mov     r13, [rsp + _gpr_save + 8*3]
 329         mov     r14, [rsp + _gpr_save + 8*4]
 330         mov     r15, [rsp + _gpr_save + 8*5]
 331 %ifndef LINUX
 332         mov     rsi, [rsp + _gpr_save + 8*6]
 333         mov     rdi, [rsp + _gpr_save + 8*7]
 334 %endif
 335         mov     rsp, [rsp + _rsp_save]  ; original SP
 336         ret
 337
 338 %ifdef LINUX
 339 section .note.GNU-stack noalloc noexec nowrite progbits
 340 %endif