ceph/src/spdk/intel-ipsec-mb/avx/mb_mgr_hmac_submit_avx.asm

   1 ;;
   2 ;; Copyright (c) 2012-2018, Intel Corporation
   3 ;;
   4 ;; Redistribution and use in source and binary forms, with or without
   5 ;; modification, are permitted provided that the following conditions are met:
   6 ;;
   7 ;;     * Redistributions of source code must retain the above copyright notice,
   8 ;;       this list of conditions and the following disclaimer.
   9 ;;     * Redistributions in binary form must reproduce the above copyright
  10 ;;       notice, this list of conditions and the following disclaimer in the
  11 ;;       documentation and/or other materials provided with the distribution.
  12 ;;     * Neither the name of Intel Corporation nor the names of its contributors
  13 ;;       may be used to endorse or promote products derived from this software
  14 ;;       without specific prior written permission.
  15 ;;
  16 ;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  17 ;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18 ;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  19 ;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
  20 ;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21 ;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
  22 ;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
  23 ;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  24 ;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  25 ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  26 ;;
  27
  28 %include "os.asm"
  29 %include "job_aes_hmac.asm"
  30 %include "mb_mgr_datastruct.asm"
  31 %include "reg_sizes.asm"
  32 %include "memcpy.asm"
  33
  34 extern sha1_mult_avx
  35
  36 section .data
  37 default rel
  38 align 16
  39 byteswap:       ;ddq 0x0c0d0e0f08090a0b0405060700010203
  40         dq 0x0405060700010203, 0x0c0d0e0f08090a0b
  41
  42 section .text
  43
  44 %if 1
  45 %ifdef LINUX
  46 %define arg1    rdi
  47 %define arg2    rsi
  48 %define reg3    rcx
  49 %define reg4    rdx
  50 %else
  51 %define arg1    rcx
  52 %define arg2    rdx
  53 %define reg3    rdi
  54 %define reg4    rsi
  55 %endif
  56
  57 %define state   arg1
  58 %define job     arg2
  59 %define len2    arg2
  60
  61
  62 ; idx needs to be in rbx, rbp, r12-r15
  63 %define last_len        rbp
  64 %define idx             rbp
  65
  66 %define p               r11
  67 %define start_offset    r11
  68
  69 %define unused_lanes    rbx
  70 %define tmp4            rbx
  71
  72 %define job_rax         rax
  73 %define len             rax
  74
  75 %define size_offset     reg3
  76 %define tmp2            reg3
  77
  78 %define lane            reg4
  79 %define tmp3            reg4
  80
  81 %define extra_blocks    r8
  82
  83 %define tmp             r9
  84 %define p2              r9
  85
  86 %define lane_data       r10
  87
  88 %endif
  89
  90 ; This routine clobbers rdi, rsi, rbx, rbp
  91 struc STACK
  92 _gpr_save:      resq    4
  93 _rsp_save:      resq    1
  94 endstruc
  95
  96 ; JOB* submit_job_hmac_avx(MB_MGR_HMAC_SHA_1_OOO *state, JOB_AES_HMAC *job)
  97 ; arg 1 : rcx : state
  98 ; arg 2 : rdx : job
  99 MKGLOBAL(submit_job_hmac_avx,function,internal)
 100 submit_job_hmac_avx:
 101
 102         mov     rax, rsp
 103         sub     rsp, STACK_size
 104         and     rsp, -16
 105
 106         mov     [rsp + _gpr_save + 8*0], rbx
 107         mov     [rsp + _gpr_save + 8*1], rbp
 108 %ifndef LINUX
 109         mov     [rsp + _gpr_save + 8*2], rsi
 110         mov     [rsp + _gpr_save + 8*3], rdi
 111 %endif
 112         mov     [rsp + _rsp_save], rax  ; original SP
 113
 114         mov     unused_lanes, [state + _unused_lanes]
 115         movzx   lane, BYTE(unused_lanes)
 116         shr     unused_lanes, 8
 117         imul    lane_data, lane, _HMAC_SHA1_LANE_DATA_size
 118         lea     lane_data, [state + _ldata + lane_data]
 119         mov     [state + _unused_lanes], unused_lanes
 120         mov     len, [job + _msg_len_to_hash_in_bytes]
 121         mov     tmp, len
 122         shr     tmp, 6  ; divide by 64, len in terms of blocks
 123
 124         mov     [lane_data + _job_in_lane], job
 125         mov     dword [lane_data + _outer_done], 0
 126         mov     [state + _lens + 2*lane], WORD(tmp)
 127
 128         mov     last_len, len
 129         and     last_len, 63
 130         lea     extra_blocks, [last_len + 9 + 63]
 131         shr     extra_blocks, 6
 132         mov     [lane_data + _extra_blocks], DWORD(extra_blocks)
 133
 134         mov     p, [job + _src]
 135         add     p, [job + _hash_start_src_offset_in_bytes]
 136         mov     [state + _args_data_ptr + PTR_SZ*lane], p
 137         cmp     len, 64
 138         jb      copy_lt64
 139
 140 fast_copy:
 141         add     p, len
 142         vmovdqu xmm0, [p - 64 + 0*16]
 143         vmovdqu xmm1, [p - 64 + 1*16]
 144         vmovdqu xmm2, [p - 64 + 2*16]
 145         vmovdqu xmm3, [p - 64 + 3*16]
 146         vmovdqa [lane_data + _extra_block + 0*16], xmm0
 147         vmovdqa [lane_data + _extra_block + 1*16], xmm1
 148         vmovdqa [lane_data + _extra_block + 2*16], xmm2
 149         vmovdqa [lane_data + _extra_block + 3*16], xmm3
 150 end_fast_copy:
 151
 152         mov     size_offset, extra_blocks
 153         shl     size_offset, 6
 154         sub     size_offset, last_len
 155         add     size_offset, 64-8
 156         mov     [lane_data + _size_offset], DWORD(size_offset)
 157         mov     start_offset, 64
 158         sub     start_offset, last_len
 159         mov     [lane_data + _start_offset], DWORD(start_offset)
 160
 161         lea     tmp, [8*64 + 8*len]
 162         bswap   tmp
 163         mov     [lane_data + _extra_block + size_offset], tmp
 164
 165         mov     tmp, [job + _auth_key_xor_ipad]
 166         vmovdqu xmm0, [tmp]
 167         mov     DWORD(tmp),  [tmp + 4*4]
 168         vmovd   [state + _args_digest + SHA1_DIGEST_WORD_SIZE*lane + 0*SHA1_DIGEST_ROW_SIZE], xmm0
 169         vpextrd [state + _args_digest + SHA1_DIGEST_WORD_SIZE*lane + 1*SHA1_DIGEST_ROW_SIZE], xmm0, 1
 170         vpextrd [state + _args_digest + SHA1_DIGEST_WORD_SIZE*lane + 2*SHA1_DIGEST_ROW_SIZE], xmm0, 2
 171         vpextrd [state + _args_digest + SHA1_DIGEST_WORD_SIZE*lane + 3*SHA1_DIGEST_ROW_SIZE], xmm0, 3
 172         mov     [state + _args_digest + SHA1_DIGEST_WORD_SIZE*lane + 4*SHA1_DIGEST_ROW_SIZE], DWORD(tmp)
 173
 174         test    len, ~63
 175         jnz     ge64_bytes
 176
 177 lt64_bytes:
 178         mov     [state + _lens + 2*lane], WORD(extra_blocks)
 179         lea     tmp, [lane_data + _extra_block + start_offset]
 180         mov     [state + _args_data_ptr + PTR_SZ*lane], tmp
 181         mov     dword [lane_data + _extra_blocks], 0
 182
 183 ge64_bytes:
 184         cmp     unused_lanes, 0xff
 185         jne     return_null
 186         jmp     start_loop
 187
 188         align   16
 189 start_loop:
 190         ; Find min length
 191         vmovdqa xmm0, [state + _lens]
 192         vphminposuw     xmm1, xmm0
 193         vpextrw DWORD(len2), xmm1, 0    ; min value
 194         vpextrw DWORD(idx), xmm1, 1     ; min index (0...3)
 195         cmp     len2, 0
 196         je      len_is_0
 197
 198         vpshuflw xmm1, xmm1, 0
 199         vpsubw  xmm0, xmm0, xmm1
 200         vmovdqa [state + _lens], xmm0
 201
 202         ; "state" and "args" are the same address, arg1
 203         ; len is arg2
 204         call    sha1_mult_avx
 205         ; state and idx are intact
 206
 207 len_is_0:
 208         ; process completed job "idx"
 209         imul    lane_data, idx, _HMAC_SHA1_LANE_DATA_size
 210         lea     lane_data, [state + _ldata + lane_data]
 211         mov     DWORD(extra_blocks), [lane_data + _extra_blocks]
 212         cmp     extra_blocks, 0
 213         jne     proc_extra_blocks
 214         cmp     dword [lane_data + _outer_done], 0
 215         jne     end_loop
 216
 217 proc_outer:
 218         mov     dword [lane_data + _outer_done], 1
 219         mov     DWORD(size_offset), [lane_data + _size_offset]
 220         mov     qword [lane_data + _extra_block + size_offset], 0
 221         mov     word [state + _lens + 2*idx], 1
 222         lea     tmp, [lane_data + _outer_block]
 223         mov     job, [lane_data + _job_in_lane]
 224         mov     [state + _args_data_ptr + PTR_SZ*idx], tmp
 225
 226         vmovd   xmm0, [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 0*SHA1_DIGEST_ROW_SIZE]
 227         vpinsrd xmm0, xmm0, [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 1*SHA1_DIGEST_ROW_SIZE], 1
 228         vpinsrd xmm0, xmm0, [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 2*SHA1_DIGEST_ROW_SIZE], 2
 229         vpinsrd xmm0, xmm0, [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 3*SHA1_DIGEST_ROW_SIZE], 3
 230         vpshufb xmm0, xmm0, [rel byteswap]
 231         mov     DWORD(tmp),  [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 4*SHA1_DIGEST_ROW_SIZE]
 232         bswap   DWORD(tmp)
 233         vmovdqa [lane_data + _outer_block], xmm0
 234         mov     [lane_data + _outer_block + 4*SHA1_DIGEST_WORD_SIZE], DWORD(tmp)
 235
 236         mov     tmp, [job + _auth_key_xor_opad]
 237         vmovdqu xmm0, [tmp]
 238         mov     DWORD(tmp),  [tmp + 4*4]
 239         vmovd   [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 0*SHA1_DIGEST_ROW_SIZE], xmm0
 240         vpextrd [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 1*SHA1_DIGEST_ROW_SIZE], xmm0, 1
 241         vpextrd [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 2*SHA1_DIGEST_ROW_SIZE], xmm0, 2
 242         vpextrd [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 3*SHA1_DIGEST_ROW_SIZE], xmm0, 3
 243         mov     [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 4*SHA1_DIGEST_ROW_SIZE], DWORD(tmp)
 244         jmp     start_loop
 245
 246         align   16
 247 proc_extra_blocks:
 248         mov     DWORD(start_offset), [lane_data + _start_offset]
 249         mov     [state + _lens + 2*idx], WORD(extra_blocks)
 250         lea     tmp, [lane_data + _extra_block + start_offset]
 251         mov     [state + _args_data_ptr + PTR_SZ*idx], tmp
 252         mov     dword [lane_data + _extra_blocks], 0
 253         jmp     start_loop
 254
 255         align   16
 256 copy_lt64:
 257         ;; less than one message block of data
 258         ;; beginning of source block
 259         ;; destination extrablock but backwards by len from where 0x80 pre-populated
 260         lea     p2, [lane_data + _extra_block  + 64]
 261         sub     p2, len
 262         memcpy_avx_64_1 p2, p, len, tmp4, tmp2, xmm0, xmm1, xmm2, xmm3
 263         mov     unused_lanes, [state + _unused_lanes]
 264         jmp     end_fast_copy
 265
 266 return_null:
 267         xor     job_rax, job_rax
 268         jmp     return
 269
 270         align   16
 271 end_loop:
 272         mov     job_rax, [lane_data + _job_in_lane]
 273         mov     unused_lanes, [state + _unused_lanes]
 274         mov     qword [lane_data + _job_in_lane], 0
 275         or      dword [job_rax + _status], STS_COMPLETED_HMAC
 276         shl     unused_lanes, 8
 277         or      unused_lanes, idx
 278         mov     [state + _unused_lanes], unused_lanes
 279
 280         mov     p, [job_rax + _auth_tag_output]
 281
 282         ; copy 12 bytes
 283         mov     DWORD(tmp),  [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 0*SHA1_DIGEST_ROW_SIZE]
 284         mov     DWORD(tmp2), [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 1*SHA1_DIGEST_ROW_SIZE]
 285         mov     DWORD(tmp3), [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 2*SHA1_DIGEST_ROW_SIZE]
 286         bswap   DWORD(tmp)
 287         bswap   DWORD(tmp2)
 288         bswap   DWORD(tmp3)
 289         mov     [p + 0*SHA1_DIGEST_WORD_SIZE], DWORD(tmp)
 290         mov     [p + 1*SHA1_DIGEST_WORD_SIZE], DWORD(tmp2)
 291         mov     [p + 2*SHA1_DIGEST_WORD_SIZE], DWORD(tmp3)
 292
 293 return:
 294
 295         mov     rbx, [rsp + _gpr_save + 8*0]
 296         mov     rbp, [rsp + _gpr_save + 8*1]
 297 %ifndef LINUX
 298         mov     rsi, [rsp + _gpr_save + 8*2]
 299         mov     rdi, [rsp + _gpr_save + 8*3]
 300 %endif
 301         mov     rsp, [rsp + _rsp_save]  ; original SP
 302
 303         ret
 304
 305 %ifdef LINUX
 306 section .note.GNU-stack noalloc noexec nowrite progbits
 307 %endif