ceph/src/spdk/intel-ipsec-mb/avx2/mb_mgr_hmac_submit_avx2.asm

   1 ;;
   2 ;; Copyright (c) 2012-2018, Intel Corporation
   3 ;;
   4 ;; Redistribution and use in source and binary forms, with or without
   5 ;; modification, are permitted provided that the following conditions are met:
   6 ;;
   7 ;;     * Redistributions of source code must retain the above copyright notice,
   8 ;;       this list of conditions and the following disclaimer.
   9 ;;     * Redistributions in binary form must reproduce the above copyright
  10 ;;       notice, this list of conditions and the following disclaimer in the
  11 ;;       documentation and/or other materials provided with the distribution.
  12 ;;     * Neither the name of Intel Corporation nor the names of its contributors
  13 ;;       may be used to endorse or promote products derived from this software
  14 ;;       without specific prior written permission.
  15 ;;
  16 ;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  17 ;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18 ;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  19 ;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
  20 ;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21 ;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
  22 ;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
  23 ;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  24 ;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  25 ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  26 ;;
  27
  28 %include "os.asm"
  29 %include "job_aes_hmac.asm"
  30 %include "mb_mgr_datastruct.asm"
  31 %include "reg_sizes.asm"
  32 %include "memcpy.asm"
  33 ;%define DO_DBGPRINT
  34 %include "dbgprint.asm"
  35
  36 extern sha1_x8_avx2
  37
  38 section .data
  39 default rel
  40 align 16
  41 byteswap:       ;ddq 0x0c0d0e0f08090a0b0405060700010203
  42         dq 0x0405060700010203, 0x0c0d0e0f08090a0b
  43
  44 section .text
  45
  46 %if 1
  47 %ifdef LINUX
  48 %define arg1    rdi
  49 %define arg2    rsi
  50 %define reg3    rcx
  51 %define reg4    rdx
  52 %else
  53 %define arg1    rcx
  54 %define arg2    rdx
  55 %define reg3    rdi
  56 %define reg4    rsi
  57 %endif
  58
  59 %define state   arg1
  60 %define job     arg2
  61 %define len2    arg2
  62
  63
  64 ; idx needs to be in rbx, rdi, rbp
  65 %define last_len        rbp
  66 %define idx             rbp
  67
  68 %define p               r11
  69 %define start_offset    r11
  70
  71 %define unused_lanes    r12
  72 %define tmp4            r12
  73
  74 %define job_rax         rax
  75 %define len             rax
  76
  77 %define size_offset     reg3
  78 %define tmp2            reg3
  79
  80 %define lane            reg4
  81 %define tmp3            reg4
  82
  83 %define extra_blocks    r8
  84
  85 %define tmp             r9
  86 %define p2              r9
  87
  88 %define lane_data       r10
  89
  90 %endif
  91
  92 ; we clobber rsi, rdi, rbp, r12; called routine clobbers also r13-r15
  93 struc STACK
  94 _gpr_save:      resq    7
  95 _rsp_save:      resq    1
  96 endstruc
  97
  98 ; JOB* submit_job_hmac_avx(MB_MGR_HMAC_SHA_1_OOO *state, JOB_AES_HMAC *job)
  99 ; arg 1 : rcx : state
 100 ; arg 2 : rdx : job
 101 MKGLOBAL(submit_job_hmac_avx2,function,internal)
 102 submit_job_hmac_avx2:
 103
 104         mov     rax, rsp
 105         sub     rsp, STACK_size
 106         and     rsp, -32                ; align to 32 byte boundary
 107         mov     [rsp + _gpr_save + 8*0], rbp
 108         mov     [rsp + _gpr_save + 8*1], r12
 109         mov     [rsp + _gpr_save + 8*2], r13
 110         mov     [rsp + _gpr_save + 8*3], r14
 111         mov     [rsp + _gpr_save + 8*4], r15
 112 %ifndef LINUX
 113         mov     [rsp + _gpr_save + 8*5], rsi
 114         mov     [rsp + _gpr_save + 8*6], rdi
 115 %endif
 116         mov     [rsp + _rsp_save], rax
 117         DBGPRINTL "---------- enter sha1 submit -----------"
 118
 119         mov     unused_lanes, [state + _unused_lanes]
 120         mov     lane, unused_lanes
 121         and     lane, 0xF           ;; just a nibble
 122         shr     unused_lanes, 4
 123         imul    lane_data, lane, _HMAC_SHA1_LANE_DATA_size
 124         lea     lane_data, [state + _ldata + lane_data]
 125         mov     [state + _unused_lanes], unused_lanes
 126
 127         mov     len, [job + _msg_len_to_hash_in_bytes]
 128         mov     tmp, len
 129         shr     tmp, 6  ; divide by 64, len in terms of blocks
 130
 131         mov     [lane_data + _job_in_lane], job
 132         mov     dword [lane_data + _outer_done], 0
 133         mov     [state + _lens + 2*lane], WORD(tmp)
 134
 135         mov     last_len, len
 136         and     last_len, 63
 137         lea     extra_blocks, [last_len + 9 + 63]
 138         shr     extra_blocks, 6
 139         mov     [lane_data + _extra_blocks], DWORD(extra_blocks)
 140
 141         mov     p, [job + _src]
 142         add     p, [job + _hash_start_src_offset_in_bytes]
 143         mov     [state + _args_data_ptr + PTR_SZ*lane], p
 144         cmp     len, 64
 145         jb      copy_lt64
 146
 147 fast_copy:
 148         add     p, len
 149         vmovdqu    ymm0, [p - 64 + 0 * 32]
 150         vmovdqu    ymm1, [p - 64 + 1 * 32]
 151         vmovdqu    [lane_data + _extra_block + 0*32], ymm0
 152         vmovdqu    [lane_data + _extra_block + 1*32], ymm1
 153 end_fast_copy:
 154
 155         mov     size_offset, extra_blocks
 156         shl     size_offset, 6
 157         sub     size_offset, last_len
 158         add     size_offset, 64-8
 159         mov     [lane_data + _size_offset], DWORD(size_offset)
 160         mov     start_offset, 64
 161         sub     start_offset, last_len
 162         mov     [lane_data + _start_offset], DWORD(start_offset)
 163
 164         lea     tmp, [8*64 + 8*len]
 165         bswap   tmp
 166         mov     [lane_data + _extra_block + size_offset], tmp
 167
 168         mov     tmp, [job + _auth_key_xor_ipad]
 169         vmovdqu xmm0, [tmp]
 170         mov     DWORD(tmp),  [tmp + 4*4]
 171         vmovd   [state + _args_digest + SHA1_DIGEST_WORD_SIZE*lane + 0*SHA1_DIGEST_ROW_SIZE], xmm0
 172         vpextrd [state + _args_digest + SHA1_DIGEST_WORD_SIZE*lane + 1*SHA1_DIGEST_ROW_SIZE], xmm0, 1
 173         vpextrd [state + _args_digest + SHA1_DIGEST_WORD_SIZE*lane + 2*SHA1_DIGEST_ROW_SIZE], xmm0, 2
 174         vpextrd [state + _args_digest + SHA1_DIGEST_WORD_SIZE*lane + 3*SHA1_DIGEST_ROW_SIZE], xmm0, 3
 175         mov     [state + _args_digest + SHA1_DIGEST_WORD_SIZE*lane + 4*SHA1_DIGEST_ROW_SIZE], DWORD(tmp)
 176
 177         test    len, ~63
 178         jnz     ge64_bytes
 179
 180 lt64_bytes:
 181         mov     [state + _lens + 2*lane], WORD(extra_blocks)
 182         lea     tmp, [lane_data + _extra_block + start_offset]
 183         mov     [state + _args_data_ptr + PTR_SZ*lane], tmp
 184         mov     dword [lane_data + _extra_blocks], 0
 185
 186 ge64_bytes:
 187         cmp     unused_lanes, 0xf
 188         jne     return_null
 189         jmp     start_loop
 190
 191         align   16
 192 start_loop:
 193         ; Find min length
 194         vmovdqa xmm0, [state + _lens]
 195         vphminposuw     xmm1, xmm0
 196         vpextrw DWORD(len2), xmm1, 0    ; min value
 197         vpextrw DWORD(idx), xmm1, 1     ; min index (0...3)
 198                 DBGPRINTL64 "min_length", len2
 199                 DBGPRINTL64 "min_length index ", idx
 200         cmp     len2, 0
 201         je      len_is_0
 202
 203         vpbroadcastw    xmm1, xmm1
 204         DBGPRINTL_XMM "SUBMIT lens after shuffle", xmm1
 205
 206         vpsubw  xmm0, xmm0, xmm1
 207         vmovdqa [state + _lens], xmm0
 208         DBGPRINTL_XMM "lengths after subtraction", xmm0
 209
 210         ; "state" and "args" are the same address, arg1
 211         ; len is arg2
 212         call    sha1_x8_avx2
 213         ; state and idx are intact
 214
 215 len_is_0:
 216         ; process completed job "idx"
 217         imul    lane_data, idx, _HMAC_SHA1_LANE_DATA_size
 218         lea     lane_data, [state + _ldata + lane_data]
 219         mov     DWORD(extra_blocks), [lane_data + _extra_blocks]
 220         cmp     extra_blocks, 0
 221         jne     proc_extra_blocks
 222         cmp     dword [lane_data + _outer_done], 0
 223         jne     end_loop
 224
 225 proc_outer:
 226         mov     dword [lane_data + _outer_done], 1
 227         mov     DWORD(size_offset), [lane_data + _size_offset]
 228         mov     qword [lane_data + _extra_block + size_offset], 0
 229         mov     word [state + _lens + 2*idx], 1
 230         lea     tmp, [lane_data + _outer_block]
 231         mov     job, [lane_data + _job_in_lane]
 232         mov     [state + _args_data_ptr + PTR_SZ*idx], tmp
 233
 234         vmovd   xmm0, [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 0*SHA1_DIGEST_ROW_SIZE]
 235         vpinsrd xmm0, xmm0, [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 1*SHA1_DIGEST_ROW_SIZE], 1
 236         vpinsrd xmm0, xmm0, [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 2*SHA1_DIGEST_ROW_SIZE], 2
 237         vpinsrd xmm0, xmm0, [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 3*SHA1_DIGEST_ROW_SIZE], 3
 238         vpshufb xmm0, xmm0, [rel byteswap]
 239         mov     DWORD(tmp),  [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 4*SHA1_DIGEST_ROW_SIZE]
 240         bswap   DWORD(tmp)
 241         vmovdqa [lane_data + _outer_block], xmm0
 242         mov     [lane_data + _outer_block + 4*SHA1_DIGEST_WORD_SIZE], DWORD(tmp)
 243
 244         mov     tmp, [job + _auth_key_xor_opad]
 245         vmovdqu xmm0, [tmp]
 246         mov     DWORD(tmp),  [tmp + 4*4]
 247         vmovd   [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 0*SHA1_DIGEST_ROW_SIZE], xmm0
 248         vpextrd [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 1*SHA1_DIGEST_ROW_SIZE], xmm0, 1
 249         vpextrd [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 2*SHA1_DIGEST_ROW_SIZE], xmm0, 2
 250         vpextrd [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 3*SHA1_DIGEST_ROW_SIZE], xmm0, 3
 251         mov     [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 4*SHA1_DIGEST_ROW_SIZE], DWORD(tmp)
 252         jmp     start_loop
 253
 254         align   16
 255 proc_extra_blocks:
 256         mov     DWORD(start_offset), [lane_data + _start_offset]
 257         mov     [state + _lens + 2*idx], WORD(extra_blocks)
 258         lea     tmp, [lane_data + _extra_block + start_offset]
 259         mov     [state + _args_data_ptr + PTR_SZ*idx], tmp
 260         mov     dword [lane_data + _extra_blocks], 0
 261         jmp     start_loop
 262
 263         align   16
 264 copy_lt64:
 265         ;; less than one message block of data
 266         ;; beginning of source block
 267         ;; destination extrablock but backwards by len from where 0x80 pre-populated
 268         lea     p2, [lane_data + _extra_block  + 64]
 269         sub     p2, len
 270         memcpy_avx2_64_1 p2, p, len, tmp4, tmp2, ymm0, ymm1
 271         mov     unused_lanes, [state + _unused_lanes]
 272         jmp     end_fast_copy
 273
 274 return_null:
 275         xor     job_rax, job_rax
 276         jmp     return
 277
 278         align   16
 279 end_loop:
 280         mov     job_rax, [lane_data + _job_in_lane]
 281         mov     unused_lanes, [state + _unused_lanes]
 282         mov     qword [lane_data + _job_in_lane], 0
 283         or      dword [job_rax + _status], STS_COMPLETED_HMAC
 284         shl     unused_lanes, 4
 285         or      unused_lanes, idx
 286         mov     [state + _unused_lanes], unused_lanes
 287
 288         mov     p, [job_rax + _auth_tag_output]
 289
 290         ; copy 12 bytes
 291         mov     DWORD(tmp),  [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 0*SHA1_DIGEST_ROW_SIZE]
 292         mov     DWORD(tmp2), [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 1*SHA1_DIGEST_ROW_SIZE]
 293         mov     DWORD(tmp3), [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 2*SHA1_DIGEST_ROW_SIZE]
 294         bswap   DWORD(tmp)
 295         bswap   DWORD(tmp2)
 296         bswap   DWORD(tmp3)
 297         mov     [p + 0*SHA1_DIGEST_WORD_SIZE], DWORD(tmp)
 298         mov     [p + 1*SHA1_DIGEST_WORD_SIZE], DWORD(tmp2)
 299         mov     [p + 2*SHA1_DIGEST_WORD_SIZE], DWORD(tmp3)
 300         vzeroupper
 301 return:
 302         DBGPRINTL "---------- exit sha1 submit -----------"
 303         mov     rbp, [rsp + _gpr_save + 8*0]
 304         mov     r12, [rsp + _gpr_save + 8*1]
 305         mov     r13, [rsp + _gpr_save + 8*2]
 306         mov     r14, [rsp + _gpr_save + 8*3]
 307         mov     r15, [rsp + _gpr_save + 8*4]
 308 %ifndef LINUX
 309         mov     rsi, [rsp + _gpr_save + 8*5]
 310         mov     rdi, [rsp + _gpr_save + 8*6]
 311 %endif
 312         mov     rsp, [rsp + _rsp_save]
 313
 314         ret
 315
 316 %ifdef LINUX
 317 section .note.GNU-stack noalloc noexec nowrite progbits
 318 %endif