ceph/src/spdk/intel-ipsec-mb/avx512/mb_mgr_hmac_submit_avx512.asm

   1 ;;
   2 ;; Copyright (c) 2012-2018, Intel Corporation
   3 ;;
   4 ;; Redistribution and use in source and binary forms, with or without
   5 ;; modification, are permitted provided that the following conditions are met:
   6 ;;
   7 ;;     * Redistributions of source code must retain the above copyright notice,
   8 ;;       this list of conditions and the following disclaimer.
   9 ;;     * Redistributions in binary form must reproduce the above copyright
  10 ;;       notice, this list of conditions and the following disclaimer in the
  11 ;;       documentation and/or other materials provided with the distribution.
  12 ;;     * Neither the name of Intel Corporation nor the names of its contributors
  13 ;;       may be used to endorse or promote products derived from this software
  14 ;;       without specific prior written permission.
  15 ;;
  16 ;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  17 ;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18 ;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  19 ;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
  20 ;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21 ;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
  22 ;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
  23 ;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  24 ;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  25 ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  26 ;;
  27
  28 ;; In System V AMD64 ABI
  29 ;;      calle saves: RBX, RBP, R12-R15
  30 ;; Windows x64 ABI
  31 ;;      calle saves: RBX, RBP, RDI, RSI, RSP, R12-R15
  32 ;;
  33 ;; Registers:           RAX RBX RCX RDX RBP RSI RDI R8  R9  R10 R11 R12 R13 R14 R15
  34 ;;                      -----------------------------------------------------------
  35 ;; Windows clobbers:    RAX     RCX RDX             R8  R9  R10 R11
  36 ;; Windows preserves:       RBX         RBP RSI RDI                 R12 R13 R14 R15
  37 ;;                      -----------------------------------------------------------
  38 ;; Linux clobbers:      RAX     RCX RDX     RSI RDI R8  R9  R10 R11
  39 ;; Linux preserves:         RBX         RBP                         R12 R13 R14 R15
  40 ;;                      -----------------------------------------------------------
  41 ;; Clobbers ZMM0-31
  42
  43 %include "os.asm"
  44 %include "job_aes_hmac.asm"
  45 %include "mb_mgr_datastruct.asm"
  46 %include "reg_sizes.asm"
  47 %include "memcpy.asm"
  48
  49 ;; %define DO_DBGPRINT
  50 %include "dbgprint.asm"
  51
  52 extern sha1_x16_avx512
  53
  54 section .data
  55 default rel
  56
  57 align 16
  58 byteswap:
  59         dq 0x0405060700010203
  60         dq 0x0c0d0e0f08090a0b
  61
  62 section .text
  63
  64 %if 1
  65 %ifdef LINUX
  66 %define arg1    rdi
  67 %define arg2    rsi
  68 %define reg3    rcx
  69 %define reg4    rdx
  70 %else
  71 %define arg1    rcx
  72 %define arg2    rdx
  73 %define reg3    rdi
  74 %define reg4    rsi
  75 %endif
  76
  77 %define state   arg1
  78 %define job     arg2
  79 %define len2    arg2
  80
  81
  82 ; idx needs to be in rbx, rdi, rbp
  83 %define last_len        rbp
  84 %define idx             rbp
  85
  86 %define p               r11
  87 %define start_offset    r11
  88
  89 %define unused_lanes    r12
  90 %define tmp4            r12
  91
  92 %define job_rax         rax
  93 %define len             rax
  94
  95 %define size_offset     reg3
  96 %define tmp2            reg3
  97
  98 %define lane            reg4
  99 %define tmp3            reg4
 100
 101 %define extra_blocks    r8
 102
 103 %define tmp             r9
 104 %define p2              r9
 105
 106 %define lane_data       r10
 107 %define num_lanes_inuse r12
 108 %define len_upper       r13
 109 %define idx_upper       r14
 110 %endif
 111
 112 ; we clobber rsi, rdi, rbp, r12; called routine clobbers also r9-r15
 113 struc STACK
 114 _gpr_save:      resq    7
 115 _rsp_save:      resq    1
 116 endstruc
 117
 118 ; JOB* submit_job_hmac_avx(MB_MGR_HMAC_SHA_1_OOO *state, JOB_AES_HMAC *job)
 119 ; arg 1 : rcx : state
 120 ; arg 2 : rdx : job
 121 MKGLOBAL(submit_job_hmac_avx512,function,internal)
 122 submit_job_hmac_avx512:
 123
 124         mov     rax, rsp
 125         sub     rsp, STACK_size
 126         and     rsp, -32                ; align to 32 byte boundary
 127         mov     [rsp + _gpr_save + 8*0], rbp
 128         mov     [rsp + _gpr_save + 8*1], r12
 129         mov     [rsp + _gpr_save + 8*2], r13
 130         mov     [rsp + _gpr_save + 8*3], r14
 131         mov     [rsp + _gpr_save + 8*4], r15
 132 %ifndef LINUX
 133         mov     [rsp + _gpr_save + 8*5], rsi
 134         mov     [rsp + _gpr_save + 8*6], rdi
 135 %endif
 136         mov     [rsp + _rsp_save], rax
 137         DBGPRINTL "---------- enter sha1 submit -----------"
 138
 139         mov     unused_lanes, [state + _unused_lanes]
 140         mov     lane, unused_lanes
 141         and     lane, 0xF           ;; just a nibble
 142         shr     unused_lanes, 4
 143         imul    lane_data, lane, _HMAC_SHA1_LANE_DATA_size
 144         lea     lane_data, [state + _ldata + lane_data]
 145         mov     [state + _unused_lanes], unused_lanes
 146         DBGPRINTL64 "lane", lane
 147         DBGPRINTL64 "unused_lanes", unused_lanes
 148
 149         add     dword [state + _num_lanes_inuse_sha1], 1
 150
 151         mov     len, [job + _msg_len_to_hash_in_bytes]
 152         mov     tmp, len
 153         shr     tmp, 6  ; divide by 64, len in terms of blocks
 154
 155         mov     [lane_data + _job_in_lane], job
 156         mov     dword [lane_data + _outer_done], 0
 157         mov     [state + _lens + 2*lane], WORD(tmp)
 158
 159         mov     last_len, len
 160         DBGPRINTL64 "last_len", last_len
 161         and     last_len, 63
 162         lea     extra_blocks, [last_len + 9 + 63]
 163         shr     extra_blocks, 6
 164         DBGPRINTL64 "extra_blocks", extra_blocks
 165         mov     [lane_data + _extra_blocks], DWORD(extra_blocks)
 166
 167         mov     p, [job + _src]
 168         add     p, [job + _hash_start_src_offset_in_bytes]
 169         mov     [state + _args_data_ptr + PTR_SZ*lane], p
 170         cmp     len, 64
 171         jb      copy_lt64
 172
 173 fast_copy:
 174         vmovdqu32       zmm0, [p - 64 + len]
 175         vmovdqu32       [lane_data + _extra_block], zmm0
 176
 177 end_fast_copy:
 178         mov     size_offset, extra_blocks
 179         shl     size_offset, 6
 180         sub     size_offset, last_len
 181         add     size_offset, 64-8
 182         mov     [lane_data + _size_offset], DWORD(size_offset)
 183         mov     start_offset, 64
 184         sub     start_offset, last_len
 185         mov     [lane_data + _start_offset], DWORD(start_offset)
 186
 187         lea     tmp, [8*64 + 8*len]
 188         bswap   tmp
 189         mov     [lane_data + _extra_block + size_offset], tmp
 190
 191         mov     tmp, [job + _auth_key_xor_ipad]
 192         vmovdqu xmm0, [tmp]
 193         mov     DWORD(tmp),  [tmp + 4*4]
 194         vmovd   [state + _args_digest + SHA1_DIGEST_WORD_SIZE*lane + 0*SHA1_DIGEST_ROW_SIZE], xmm0
 195         vpextrd [state + _args_digest + SHA1_DIGEST_WORD_SIZE*lane + 1*SHA1_DIGEST_ROW_SIZE], xmm0, 1
 196         vpextrd [state + _args_digest + SHA1_DIGEST_WORD_SIZE*lane + 2*SHA1_DIGEST_ROW_SIZE], xmm0, 2
 197         vpextrd [state + _args_digest + SHA1_DIGEST_WORD_SIZE*lane + 3*SHA1_DIGEST_ROW_SIZE], xmm0, 3
 198         mov     [state + _args_digest + SHA1_DIGEST_WORD_SIZE*lane + 4*SHA1_DIGEST_ROW_SIZE], DWORD(tmp)
 199
 200         test    len, ~63
 201         jnz     ge64_bytes
 202
 203 lt64_bytes:
 204         DBGPRINTL64 "lt64_bytes extra_blocks", extra_blocks
 205         DBGPRINTL64 "lt64_bytes start_offset", start_offset
 206         mov     [state + _lens + 2*lane], WORD(extra_blocks)
 207         lea     tmp, [lane_data + _extra_block + start_offset]
 208         mov     [state + _args_data_ptr + PTR_SZ*lane], tmp
 209         mov     dword [lane_data + _extra_blocks], 0
 210
 211 ge64_bytes:
 212         mov     DWORD(num_lanes_inuse), [state + _num_lanes_inuse_sha1]
 213         cmp     num_lanes_inuse, 0x10 ; all 16 lanes used?
 214         jne     return_null
 215         jmp     start_loop
 216
 217         align   16
 218 start_loop:
 219         ; Find min length
 220         vmovdqa xmm0, [state + _lens]
 221         vphminposuw     xmm1, xmm0
 222         vpextrw DWORD(len2), xmm1, 0    ; min value
 223         vpextrw DWORD(idx), xmm1, 1     ; min index (0...7)
 224
 225         vmovdqa xmm2, [state + _lens + 8*2]
 226         vphminposuw     xmm3, xmm2
 227         vpextrw DWORD(len_upper), xmm3, 0       ; min value
 228         vpextrw DWORD(idx_upper), xmm3, 1       ; min index (8...F)
 229
 230         cmp     len2, len_upper
 231         jle     use_min
 232
 233         vmovdqa xmm1, xmm3
 234         mov     len2, len_upper
 235         mov     idx, idx_upper  ; idx would be in range 0..7
 236         add     idx, 8          ; to reflect that index is in 8..F range
 237
 238 use_min:
 239         cmp     len2, 0
 240         je      len_is_0
 241
 242         DBGPRINTL64 "min_length", len2
 243         DBGPRINTL64 "min_length index ", idx
 244
 245         vpbroadcastw    xmm1, xmm1
 246         DBGPRINTL_XMM "SUBMIT lens after shuffle", xmm1
 247
 248         vpsubw  xmm0, xmm0, xmm1
 249         vmovdqa [state + _lens + 0*2], xmm0
 250         vpsubw  xmm2, xmm2, xmm1
 251         vmovdqa [state + _lens + 8*2], xmm2
 252         DBGPRINTL_XMM "lengths after subtraction (0..7)", xmm0
 253         DBGPRINTL_XMM "lengths after subtraction (8..F)", xmm2
 254
 255         ; "state" and "args" are the same address, arg1
 256         ; len is arg2
 257         call    sha1_x16_avx512
 258         ; state and idx are intact
 259
 260 len_is_0:
 261         ; process completed job "idx"
 262         imul    lane_data, idx, _HMAC_SHA1_LANE_DATA_size
 263         lea     lane_data, [state + _ldata + lane_data]
 264         mov     DWORD(extra_blocks), [lane_data + _extra_blocks]
 265         cmp     extra_blocks, 0
 266         jne     proc_extra_blocks
 267         cmp     dword [lane_data + _outer_done], 0
 268         jne     end_loop
 269
 270 proc_outer:
 271         mov     dword [lane_data + _outer_done], 1
 272         mov     DWORD(size_offset), [lane_data + _size_offset]
 273         mov     qword [lane_data + _extra_block + size_offset], 0
 274         mov     word [state + _lens + 2*idx], 1
 275         lea     tmp, [lane_data + _outer_block]
 276         mov     job, [lane_data + _job_in_lane]
 277         mov     [state + _args_data_ptr + PTR_SZ*idx], tmp
 278
 279         vmovd   xmm0, [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 0*SHA1_DIGEST_ROW_SIZE]
 280         vpinsrd xmm0, xmm0, [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 1*SHA1_DIGEST_ROW_SIZE], 1
 281         vpinsrd xmm0, xmm0, [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 2*SHA1_DIGEST_ROW_SIZE], 2
 282         vpinsrd xmm0, xmm0, [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 3*SHA1_DIGEST_ROW_SIZE], 3
 283         vpshufb xmm0, xmm0, [rel byteswap]
 284         mov     DWORD(tmp),  [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 4*SHA1_DIGEST_ROW_SIZE]
 285         bswap   DWORD(tmp)
 286         vmovdqa [lane_data + _outer_block], xmm0
 287         mov     [lane_data + _outer_block + 4*SHA1_DIGEST_WORD_SIZE], DWORD(tmp)
 288
 289         mov     tmp, [job + _auth_key_xor_opad]
 290         vmovdqu xmm0, [tmp]
 291         mov     DWORD(tmp),  [tmp + 4*4]
 292         vmovd   [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 0*SHA1_DIGEST_ROW_SIZE], xmm0
 293         vpextrd [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 1*SHA1_DIGEST_ROW_SIZE], xmm0, 1
 294         vpextrd [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 2*SHA1_DIGEST_ROW_SIZE], xmm0, 2
 295         vpextrd [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 3*SHA1_DIGEST_ROW_SIZE], xmm0, 3
 296         mov     [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 4*SHA1_DIGEST_ROW_SIZE], DWORD(tmp)
 297
 298         jmp     start_loop
 299
 300         align   16
 301 proc_extra_blocks:
 302         mov     DWORD(start_offset), [lane_data + _start_offset]
 303         mov     [state + _lens + 2*idx], WORD(extra_blocks)
 304         lea     tmp, [lane_data + _extra_block + start_offset]
 305         mov     [state + _args_data_ptr + PTR_SZ*idx], tmp
 306         mov     dword [lane_data + _extra_blocks], 0
 307         jmp     start_loop
 308
 309         align   16
 310 copy_lt64:
 311         ;; less than one message block of data
 312         ;; beginning of source block
 313         ;; destination extrablock but backwards by len from where 0x80 pre-populated
 314         lea     p2, [lane_data + _extra_block  + 64]
 315         sub     p2, len
 316         memcpy_avx2_64_1 p2, p, len, tmp4, tmp2, ymm0, ymm1
 317         mov     unused_lanes, [state + _unused_lanes]
 318         jmp     end_fast_copy
 319
 320 return_null:
 321         xor     job_rax, job_rax
 322         jmp     return
 323
 324         align   16
 325 end_loop:
 326         mov     job_rax, [lane_data + _job_in_lane]
 327         or      dword [job_rax + _status], STS_COMPLETED_HMAC
 328         mov     qword [lane_data + _job_in_lane], 0
 329
 330         mov     unused_lanes, [state + _unused_lanes]
 331         shl     unused_lanes, 4
 332         or      unused_lanes, idx
 333         mov     [state + _unused_lanes], unused_lanes
 334
 335         sub     dword [state + _num_lanes_inuse_sha1], 1
 336
 337         mov     p, [job_rax + _auth_tag_output]
 338
 339         vzeroupper
 340
 341         ; copy 12 bytes
 342         mov     DWORD(tmp),  [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 0*SHA1_DIGEST_ROW_SIZE]
 343         mov     DWORD(tmp2), [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 1*SHA1_DIGEST_ROW_SIZE]
 344         mov     DWORD(tmp3), [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 2*SHA1_DIGEST_ROW_SIZE]
 345         bswap   DWORD(tmp)
 346         bswap   DWORD(tmp2)
 347         bswap   DWORD(tmp3)
 348         mov     [p + 0*SHA1_DIGEST_WORD_SIZE], DWORD(tmp)
 349         mov     [p + 1*SHA1_DIGEST_WORD_SIZE], DWORD(tmp2)
 350         mov     [p + 2*SHA1_DIGEST_WORD_SIZE], DWORD(tmp3)
 351
 352         cmp     qword [job_rax + _auth_tag_output_len_in_bytes], 12
 353         je      return
 354
 355         ;; copy remaining 8 bytes to return 20 byte digest
 356         mov     DWORD(tmp),  [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 3*SHA1_DIGEST_ROW_SIZE]
 357         mov     DWORD(tmp2), [state + _args_digest + SHA1_DIGEST_WORD_SIZE*idx + 4*SHA1_DIGEST_ROW_SIZE]
 358         bswap   DWORD(tmp)
 359         bswap   DWORD(tmp2)
 360         mov     [p + 3*SHA1_DIGEST_WORD_SIZE], DWORD(tmp)
 361         mov     [p + 4*SHA1_DIGEST_WORD_SIZE], DWORD(tmp2)
 362 return:
 363         DBGPRINTL "---------- exit sha1 submit -----------"
 364         mov     rbp, [rsp + _gpr_save + 8*0]
 365         mov     r12, [rsp + _gpr_save + 8*1]
 366         mov     r13, [rsp + _gpr_save + 8*2]
 367         mov     r14, [rsp + _gpr_save + 8*3]
 368         mov     r15, [rsp + _gpr_save + 8*4]
 369 %ifndef LINUX
 370         mov     rsi, [rsp + _gpr_save + 8*5]
 371         mov     rdi, [rsp + _gpr_save + 8*6]
 372 %endif
 373         mov     rsp, [rsp + _rsp_save]
 374
 375         ret
 376
 377 %ifdef LINUX
 378 section .note.GNU-stack noalloc noexec nowrite progbits
 379 %endif