ceph/src/spdk/intel-ipsec-mb/sse/mb_mgr_hmac_sha_256_submit_sse.asm

   1 ;;
   2 ;; Copyright (c) 2012-2018, Intel Corporation
   3 ;;
   4 ;; Redistribution and use in source and binary forms, with or without
   5 ;; modification, are permitted provided that the following conditions are met:
   6 ;;
   7 ;;     * Redistributions of source code must retain the above copyright notice,
   8 ;;       this list of conditions and the following disclaimer.
   9 ;;     * Redistributions in binary form must reproduce the above copyright
  10 ;;       notice, this list of conditions and the following disclaimer in the
  11 ;;       documentation and/or other materials provided with the distribution.
  12 ;;     * Neither the name of Intel Corporation nor the names of its contributors
  13 ;;       may be used to endorse or promote products derived from this software
  14 ;;       without specific prior written permission.
  15 ;;
  16 ;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  17 ;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18 ;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  19 ;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
  20 ;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21 ;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
  22 ;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
  23 ;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  24 ;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  25 ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  26 ;;
  27
  28 %include "os.asm"
  29 %include "job_aes_hmac.asm"
  30 %include "mb_mgr_datastruct.asm"
  31 %include "reg_sizes.asm"
  32 %include "memcpy.asm"
  33
  34 extern sha_256_mult_sse
  35
  36 section .data
  37 default rel
  38 align 16
  39 byteswap:       ;ddq 0x0c0d0e0f08090a0b0405060700010203
  40         dq 0x0405060700010203, 0x0c0d0e0f08090a0b
  41
  42 section .text
  43
  44 %ifndef FUNC
  45 %define FUNC submit_job_hmac_sha_256_sse
  46 %endif
  47
  48 %if 1
  49 %ifdef LINUX
  50 %define arg1    rdi
  51 %define arg2    rsi
  52 %define reg3    rcx
  53 %define reg4    rdx
  54 %else
  55 %define arg1    rcx
  56 %define arg2    rdx
  57 %define reg3    rdi
  58 %define reg4    rsi
  59 %endif
  60
  61 %define state   arg1
  62 %define job     arg2
  63 %define len2    arg2
  64
  65
  66 ; idx needs to be in rbx, rbp, r13-r15
  67 %define last_len        rbp
  68 %define idx             rbp
  69
  70 %define p               r11
  71 %define start_offset    r11
  72
  73 %define unused_lanes    rbx
  74 %define tmp4            rbx
  75
  76 %define job_rax         rax
  77 %define len             rax
  78
  79 %define size_offset     reg3
  80 %define tmp2            reg3
  81
  82 %define lane            reg4
  83 %define tmp3            reg4
  84
  85 %define extra_blocks    r8
  86
  87 %define tmp             r9
  88 %define p2              r9
  89
  90 %define lane_data       r10
  91
  92 %endif
  93
  94 ; This routine clobbers rbx, rbp, rsi, rdi; called routine also clobbers r12
  95 struc STACK
  96 _gpr_save:      resq    5
  97 _rsp_save:      resq    1
  98 endstruc
  99
 100 ; JOB* FUNC(MB_MGR_HMAC_SHA_256_OOO *state, JOB_AES_HMAC *job)
 101 ; arg 1 : rcx : state
 102 ; arg 2 : rdx : job
 103 MKGLOBAL(FUNC,function,internal)
 104 FUNC:
 105
 106         mov     rax, rsp
 107         sub     rsp, STACK_size
 108         and     rsp, -16
 109
 110         mov     [rsp + _gpr_save + 8*0], rbx
 111         mov     [rsp + _gpr_save + 8*1], rbp
 112         mov     [rsp + _gpr_save + 8*2], r12
 113 %ifndef LINUX
 114         mov     [rsp + _gpr_save + 8*3], rsi
 115         mov     [rsp + _gpr_save + 8*4], rdi
 116 %endif
 117         mov     [rsp + _rsp_save], rax  ; original SP
 118
 119         mov     unused_lanes, [state + _unused_lanes_sha256]
 120         movzx   lane, BYTE(unused_lanes)
 121         shr     unused_lanes, 8
 122         imul    lane_data, lane, _HMAC_SHA1_LANE_DATA_size
 123         lea     lane_data, [state + _ldata_sha256 + lane_data]
 124         mov     [state + _unused_lanes_sha256], unused_lanes
 125         mov     len, [job + _msg_len_to_hash_in_bytes]
 126         mov     tmp, len
 127         shr     tmp, 6  ; divide by 64, len in terms of blocks
 128
 129         mov     [lane_data + _job_in_lane], job
 130         mov     dword [lane_data + _outer_done], 0
 131         mov     [state + _lens_sha256 + 2*lane], WORD(tmp)
 132
 133         mov     last_len, len
 134         and     last_len, 63
 135         lea     extra_blocks, [last_len + 9 + 63]
 136         shr     extra_blocks, 6
 137         mov     [lane_data + _extra_blocks], DWORD(extra_blocks)
 138
 139         mov     p, [job + _src]
 140         add     p, [job + _hash_start_src_offset_in_bytes]
 141         mov     [state + _args_data_ptr_sha256 + 8*lane], p
 142
 143         cmp     len, 64
 144         jb      copy_lt64
 145
 146 fast_copy:
 147         add             p, len
 148         movdqu  xmm0, [p - 64 + 0*16]
 149         movdqu  xmm1, [p - 64 + 1*16]
 150         movdqu  xmm2, [p - 64 + 2*16]
 151         movdqu  xmm3, [p - 64 + 3*16]
 152         movdqa  [lane_data + _extra_block + 0*16], xmm0
 153         movdqa  [lane_data + _extra_block + 1*16], xmm1
 154         movdqa  [lane_data + _extra_block + 2*16], xmm2
 155         movdqa  [lane_data + _extra_block + 3*16], xmm3
 156 end_fast_copy:
 157
 158         mov     size_offset, extra_blocks
 159         shl     size_offset, 6
 160         sub     size_offset, last_len
 161         add     size_offset, 64-8
 162         mov     [lane_data + _size_offset], DWORD(size_offset)
 163         mov     start_offset, 64
 164         sub     start_offset, last_len
 165         mov     [lane_data + _start_offset], DWORD(start_offset)
 166
 167         lea     tmp, [8*64 + 8*len]
 168         bswap   tmp
 169         mov     [lane_data + _extra_block + size_offset], tmp
 170
 171         mov     tmp, [job + _auth_key_xor_ipad]
 172         movdqu  xmm0, [tmp]
 173         movdqu  xmm1, [tmp + 4*4]
 174         movd    [state + _args_digest_sha256 + 4*lane + 0*SHA256_DIGEST_ROW_SIZE], xmm0
 175         pextrd  [state + _args_digest_sha256 + 4*lane + 1*SHA256_DIGEST_ROW_SIZE], xmm0, 1
 176         pextrd  [state + _args_digest_sha256 + 4*lane + 2*SHA256_DIGEST_ROW_SIZE], xmm0, 2
 177         pextrd  [state + _args_digest_sha256 + 4*lane + 3*SHA256_DIGEST_ROW_SIZE], xmm0, 3
 178         movd    [state + _args_digest_sha256 + 4*lane + 4*SHA256_DIGEST_ROW_SIZE], xmm1
 179         pextrd  [state + _args_digest_sha256 + 4*lane + 5*SHA256_DIGEST_ROW_SIZE], xmm1, 1
 180         pextrd  [state + _args_digest_sha256 + 4*lane + 6*SHA256_DIGEST_ROW_SIZE], xmm1, 2
 181         pextrd  [state + _args_digest_sha256 + 4*lane + 7*SHA256_DIGEST_ROW_SIZE], xmm1, 3
 182         test    len, ~63
 183         jnz     ge64_bytes
 184
 185 lt64_bytes:
 186         mov     [state + _lens_sha256 + 2*lane], WORD(extra_blocks)
 187         lea     tmp, [lane_data + _extra_block + start_offset]
 188         mov     [state + _args_data_ptr_sha256 + 8*lane], tmp
 189         mov     dword [lane_data + _extra_blocks], 0
 190
 191 ge64_bytes:
 192         cmp     unused_lanes, 0xff
 193         jne     return_null
 194         jmp     start_loop
 195
 196         align   16
 197 start_loop:
 198         ; Find min length
 199         movdqa  xmm0, [state + _lens_sha256]
 200         phminposuw      xmm1, xmm0
 201         pextrw  len2, xmm1, 0   ; min value
 202         pextrw  idx, xmm1, 1    ; min index (0...3)
 203         cmp     len2, 0
 204         je      len_is_0
 205
 206         pshuflw xmm1, xmm1, 0
 207         psubw   xmm0, xmm1
 208         movdqa  [state + _lens_sha256], xmm0
 209
 210         ; "state" and "args" are the same address, arg1
 211         ; len is arg2
 212         call    sha_256_mult_sse
 213         ; state and idx are intact
 214
 215 len_is_0:
 216         ; process completed job "idx"
 217         imul    lane_data, idx, _HMAC_SHA1_LANE_DATA_size
 218         lea     lane_data, [state + _ldata_sha256 + lane_data]
 219         mov     DWORD(extra_blocks), [lane_data + _extra_blocks]
 220         cmp     extra_blocks, 0
 221         jne     proc_extra_blocks
 222         cmp     dword [lane_data + _outer_done], 0
 223         jne     end_loop
 224
 225 proc_outer:
 226         mov     dword [lane_data + _outer_done], 1
 227         mov     DWORD(size_offset), [lane_data + _size_offset]
 228         mov     qword [lane_data + _extra_block + size_offset], 0
 229         mov     word [state + _lens_sha256 + 2*idx], 1
 230         lea     tmp, [lane_data + _outer_block]
 231         mov     job, [lane_data + _job_in_lane]
 232         mov     [state + _args_data_ptr_sha256 + 8*idx], tmp
 233
 234         movd    xmm0, [state + _args_digest_sha256 + 4*idx + 0*SHA256_DIGEST_ROW_SIZE]
 235         pinsrd  xmm0, [state + _args_digest_sha256 + 4*idx + 1*SHA256_DIGEST_ROW_SIZE], 1
 236         pinsrd  xmm0, [state + _args_digest_sha256 + 4*idx + 2*SHA256_DIGEST_ROW_SIZE], 2
 237         pinsrd  xmm0, [state + _args_digest_sha256 + 4*idx + 3*SHA256_DIGEST_ROW_SIZE], 3
 238         pshufb  xmm0, [rel byteswap]
 239         movd    xmm1, [state + _args_digest_sha256 + 4*idx + 4*SHA256_DIGEST_ROW_SIZE]
 240         pinsrd  xmm1, [state + _args_digest_sha256 + 4*idx + 5*SHA256_DIGEST_ROW_SIZE], 1
 241         pinsrd  xmm1, [state + _args_digest_sha256 + 4*idx + 6*SHA256_DIGEST_ROW_SIZE], 2
 242 %ifndef SHA224
 243         pinsrd  xmm1, [state + _args_digest_sha256 + 4*idx + 7*SHA256_DIGEST_ROW_SIZE], 3
 244 %endif
 245         pshufb  xmm1, [rel byteswap]
 246         movdqa  [lane_data + _outer_block], xmm0
 247         movdqa  [lane_data + _outer_block + 4*4], xmm1
 248 %ifdef SHA224
 249         mov             dword [lane_data + _outer_block + 7*4], 0x80
 250 %endif
 251
 252
 253         mov     tmp, [job + _auth_key_xor_opad]
 254         movdqu  xmm0, [tmp]
 255         movdqu  xmm1,  [tmp + 4*4]
 256         movd    [state + _args_digest_sha256 + 4*idx + 0*SHA256_DIGEST_ROW_SIZE], xmm0
 257         pextrd  [state + _args_digest_sha256 + 4*idx + 1*SHA256_DIGEST_ROW_SIZE], xmm0, 1
 258         pextrd  [state + _args_digest_sha256 + 4*idx + 2*SHA256_DIGEST_ROW_SIZE], xmm0, 2
 259         pextrd  [state + _args_digest_sha256 + 4*idx + 3*SHA256_DIGEST_ROW_SIZE], xmm0, 3
 260         movd    [state + _args_digest_sha256 + 4*idx + 4*SHA256_DIGEST_ROW_SIZE], xmm1
 261         pextrd  [state + _args_digest_sha256 + 4*idx + 5*SHA256_DIGEST_ROW_SIZE], xmm1, 1
 262         pextrd  [state + _args_digest_sha256 + 4*idx + 6*SHA256_DIGEST_ROW_SIZE], xmm1, 2
 263         pextrd  [state + _args_digest_sha256 + 4*idx + 7*SHA256_DIGEST_ROW_SIZE], xmm1, 3
 264         jmp     start_loop
 265
 266         align   16
 267 proc_extra_blocks:
 268         mov     DWORD(start_offset), [lane_data + _start_offset]
 269         mov     [state + _lens_sha256 + 2*idx], WORD(extra_blocks)
 270         lea     tmp, [lane_data + _extra_block + start_offset]
 271         mov     [state + _args_data_ptr_sha256 + 8*idx], tmp
 272         mov     dword [lane_data + _extra_blocks], 0
 273         jmp     start_loop
 274
 275         align   16
 276
 277 copy_lt64:
 278         ;; less than one message block of data
 279         ;; beginning of source block
 280         ;; destination extrablock but backwards by len from where 0x80 pre-populated
 281         ;; p2 clobbers unused_lanes, undo before exit
 282         lea     p2, [lane_data + _extra_block  + 64]
 283         sub     p2, len
 284         memcpy_sse_64_1 p2, p, len, tmp4, tmp2, xmm0, xmm1, xmm2, xmm3
 285         mov     unused_lanes, [state + _unused_lanes_sha256]
 286         jmp     end_fast_copy
 287
 288 return_null:
 289         xor     job_rax, job_rax
 290         jmp     return
 291
 292         align   16
 293 end_loop:
 294         mov     job_rax, [lane_data + _job_in_lane]
 295         mov     unused_lanes, [state + _unused_lanes_sha256]
 296         mov     qword [lane_data + _job_in_lane], 0
 297         or      dword [job_rax + _status], STS_COMPLETED_HMAC
 298         shl     unused_lanes, 8
 299         or      unused_lanes, idx
 300         mov     [state + _unused_lanes_sha256], unused_lanes
 301
 302         mov     p, [job_rax + _auth_tag_output]
 303
 304         ; copy 14 bytes for SHA224 and 16 bytes for SHA256
 305         mov     DWORD(tmp),  [state + _args_digest_sha256 + 4*idx + 0*SHA256_DIGEST_ROW_SIZE]
 306         mov     DWORD(tmp2), [state + _args_digest_sha256 + 4*idx + 1*SHA256_DIGEST_ROW_SIZE]
 307         mov     DWORD(tmp3), [state + _args_digest_sha256 + 4*idx + 2*SHA256_DIGEST_ROW_SIZE]
 308         mov     DWORD(tmp4), [state + _args_digest_sha256 + 4*idx + 3*SHA256_DIGEST_ROW_SIZE]
 309
 310         bswap   DWORD(tmp)
 311         bswap   DWORD(tmp2)
 312         bswap   DWORD(tmp3)
 313         bswap   DWORD(tmp4)
 314
 315         mov     [p + 0*4], DWORD(tmp)
 316         mov     [p + 1*4], DWORD(tmp2)
 317         mov     [p + 2*4], DWORD(tmp3)
 318
 319 %ifdef SHA224
 320         mov     [p + 3*4], WORD(tmp4)
 321 %else
 322         mov     [p + 3*4], DWORD(tmp4)
 323 %endif
 324
 325
 326 return:
 327
 328         mov     rbx, [rsp + _gpr_save + 8*0]
 329         mov     rbp, [rsp + _gpr_save + 8*1]
 330         mov     r12, [rsp + _gpr_save + 8*2]
 331 %ifndef LINUX
 332         mov     rsi, [rsp + _gpr_save + 8*3]
 333         mov     rdi, [rsp + _gpr_save + 8*4]
 334 %endif
 335         mov     rsp, [rsp + _rsp_save]  ; original SP
 336
 337         ret
 338
 339 %ifdef LINUX
 340 section .note.GNU-stack noalloc noexec nowrite progbits
 341 %endif