ceph/src/spdk/intel-ipsec-mb/sse/mb_mgr_hmac_sha_512_flush_sse.asm

   1 ;;
   2 ;; Copyright (c) 2012-2018, Intel Corporation
   3 ;;
   4 ;; Redistribution and use in source and binary forms, with or without
   5 ;; modification, are permitted provided that the following conditions are met:
   6 ;;
   7 ;;     * Redistributions of source code must retain the above copyright notice,
   8 ;;       this list of conditions and the following disclaimer.
   9 ;;     * Redistributions in binary form must reproduce the above copyright
  10 ;;       notice, this list of conditions and the following disclaimer in the
  11 ;;       documentation and/or other materials provided with the distribution.
  12 ;;     * Neither the name of Intel Corporation nor the names of its contributors
  13 ;;       may be used to endorse or promote products derived from this software
  14 ;;       without specific prior written permission.
  15 ;;
  16 ;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  17 ;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18 ;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  19 ;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
  20 ;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21 ;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
  22 ;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
  23 ;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  24 ;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  25 ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  26 ;;
  27
  28 %include "os.asm"
  29 %include "job_aes_hmac.asm"
  30 %include "mb_mgr_datastruct.asm"
  31 %include "reg_sizes.asm"
  32
  33 extern sha512_x2_sse
  34
  35 section .data
  36 default rel
  37 align 16
  38 byteswap:       ;ddq 0x08090a0b0c0d0e0f0001020304050607
  39         dq 0x0001020304050607, 0x08090a0b0c0d0e0f
  40 len_masks:
  41         ;ddq 0x0000000000000000000000000000FFFF
  42         dq 0x000000000000FFFF, 0x0000000000000000
  43         ;ddq 0x000000000000000000000000FFFF0000
  44         dq 0x00000000FFFF0000, 0x0000000000000000
  45 one:    dq  1
  46
  47 section .text
  48
  49 %ifndef FUNC
  50 %define FUNC flush_job_hmac_sha_512_sse
  51 %define SHA_X_DIGEST_SIZE 512
  52 %endif
  53
  54 %if 1
  55 %ifdef LINUX
  56 %define arg1    rdi
  57 %define arg2    rsi
  58 %else
  59 %define arg1    rcx
  60 %define arg2    rdx
  61 %endif
  62
  63 %define state   arg1
  64 %define job     arg2
  65 %define len2    arg2
  66
  67
  68 ; idx needs to be in rbx, rbp, r12-r15
  69 %define idx             rbp
  70
  71 %define unused_lanes    rbx
  72 %define lane_data       rbx
  73 %define tmp2            rbx
  74
  75 %define job_rax         rax
  76 %define tmp1            rax
  77 %define size_offset     rax
  78 %define tmp             rax
  79 %define start_offset    rax
  80
  81 %define tmp3            arg1
  82
  83 %define extra_blocks    arg2
  84 %define p               arg2
  85
  86 %define tmp4            r8
  87
  88 %define tmp5            r9
  89
  90 %define tmp6            r10
  91
  92 %endif
  93
  94 ; This routine clobbers rbx, rbp
  95 struc STACK
  96 _gpr_save:      resq    2
  97 _rsp_save:      resq    1
  98 endstruc
  99
 100 %define APPEND(a,b) a %+ b
 101
 102 ; JOB* FUNC(MB_MGR_HMAC_SHA_512_OOO *state)
 103 ; arg 1 : rcx : state
 104 MKGLOBAL(FUNC,function,internal)
 105 FUNC:
 106
 107         mov     rax, rsp
 108         sub     rsp, STACK_size
 109         and     rsp, -16
 110
 111         mov     [rsp + _gpr_save + 8*0], rbx
 112         mov     [rsp + _gpr_save + 8*1], rbp
 113         mov     [rsp + _rsp_save], rax  ; original SP
 114
 115         mov     unused_lanes, [state + _unused_lanes_sha512]
 116         bt      unused_lanes, 16+7
 117         jc      return_null
 118
 119         ; find a lane with a non-null job
 120         xor     idx, idx
 121         cmp     qword [state + _ldata_sha512 + 1 * _SHA512_LANE_DATA_size + _job_in_lane_sha512], 0
 122         cmovne  idx, [rel one]
 123 copy_lane_data:
 124         ; copy good lane (idx) to empty lanes
 125         movdqa  xmm0, [state + _lens_sha512]
 126         mov     tmp, [state + _args_sha512 + _data_ptr_sha512 + PTR_SZ*idx]
 127
 128 %assign I 0
 129 %rep 2
 130         cmp     qword [state + _ldata_sha512 + I * _SHA512_LANE_DATA_size + _job_in_lane_sha512], 0
 131         jne     APPEND(skip_,I)
 132         mov     [state + _args_sha512 + _data_ptr_sha512 + PTR_SZ*I], tmp
 133         por     xmm0, [rel len_masks + 16*I]
 134 APPEND(skip_,I):
 135 %assign I (I+1)
 136 %endrep
 137
 138         movdqa  [state + _lens_sha512], xmm0
 139
 140         phminposuw      xmm1, xmm0
 141         pextrw  len2, xmm1, 0   ; min value
 142         pextrw  idx, xmm1, 1    ; min index (0...3)
 143         cmp     len2, 0
 144         je      len_is_0
 145
 146         pshuflw xmm1, xmm1, 0xA0
 147         psubw   xmm0, xmm1
 148         movdqa  [state + _lens_sha512], xmm0
 149
 150         ; "state" and "args" are the same address, arg1
 151         ; len is arg2
 152         call    sha512_x2_sse
 153         ; state and idx are intact
 154
 155 len_is_0:
 156         ; process completed job "idx"
 157         imul    lane_data, idx, _SHA512_LANE_DATA_size
 158         lea     lane_data, [state + _ldata_sha512 + lane_data]
 159         mov     DWORD(extra_blocks), [lane_data + _extra_blocks_sha512]
 160         cmp     extra_blocks, 0
 161         jne     proc_extra_blocks
 162         cmp     dword [lane_data + _outer_done_sha512], 0
 163         jne     end_loop
 164
 165 proc_outer:
 166         mov     dword [lane_data + _outer_done_sha512], 1
 167         mov     DWORD(size_offset), [lane_data + _size_offset_sha512]
 168         mov     qword [lane_data + _extra_block_sha512 + size_offset], 0
 169         mov     word [state + _lens_sha512 + 2*idx], 1
 170         lea     tmp, [lane_data + _outer_block_sha512]
 171         mov     job, [lane_data + _job_in_lane_sha512]
 172         mov     [state + _args_data_ptr_sha512 + PTR_SZ*idx], tmp
 173
 174 %assign I 0
 175 %rep (SHA_X_DIGEST_SIZE / (8*16))
 176         movq    xmm0, [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + (2*I)*SHA512_DIGEST_ROW_SIZE]
 177         pinsrq  xmm0, [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + (2*I + 1) *SHA512_DIGEST_ROW_SIZE], 1
 178         pshufb  xmm0, [rel byteswap]
 179         movdqa  [lane_data + _outer_block_sha512 + I*16], xmm0
 180 %assign I (I+1)
 181 %endrep
 182
 183         mov     tmp, [job + _auth_key_xor_opad]
 184 %assign I 0
 185 %rep 4
 186         movdqu  xmm0, [tmp + I * 16]
 187         movq    [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 2*I*SHA512_DIGEST_ROW_SIZE], xmm0
 188         pextrq  [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + (2*I + 1)*SHA512_DIGEST_ROW_SIZE], xmm0, 1
 189 %assign I (I+1)
 190 %endrep
 191         jmp     copy_lane_data
 192
 193         align   16
 194 proc_extra_blocks:
 195         mov     DWORD(start_offset), [lane_data + _start_offset_sha512]
 196         mov     [state + _lens_sha512 + 2*idx], WORD(extra_blocks)
 197         lea     tmp, [lane_data + _extra_block_sha512 + start_offset]
 198         mov     [state + _args_data_ptr_sha512 + PTR_SZ*idx], tmp
 199         mov     dword [lane_data + _extra_blocks_sha512], 0
 200         jmp     copy_lane_data
 201
 202 return_null:
 203         xor     job_rax, job_rax
 204         jmp     return
 205
 206         align   16
 207 end_loop:
 208         mov     job_rax, [lane_data + _job_in_lane_sha512]
 209         mov     qword [lane_data + _job_in_lane_sha512], 0
 210         or      dword [job_rax + _status], STS_COMPLETED_HMAC
 211         mov     unused_lanes, [state + _unused_lanes_sha512]
 212         shl     unused_lanes, 8
 213         or      unused_lanes, idx
 214         mov     [state + _unused_lanes_sha512], unused_lanes
 215
 216         mov     p, [job_rax + _auth_tag_output]
 217
 218 %if (SHA_X_DIGEST_SIZE != 384)
 219         cmp     qword [job_rax + _auth_tag_output_len_in_bytes], 32
 220         jne     copy_full_digest
 221 %else
 222         cmp     qword [job_rax + _auth_tag_output_len_in_bytes], 24
 223         jne     copy_full_digest
 224 %endif
 225         ;; copy 32 bytes for SHA512 // 24 bytes for SHA384
 226         mov     QWORD(tmp2), [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 0*SHA512_DIGEST_ROW_SIZE]
 227         mov     QWORD(tmp4), [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 1*SHA512_DIGEST_ROW_SIZE]
 228         mov     QWORD(tmp6), [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 2*SHA512_DIGEST_ROW_SIZE]
 229 %if (SHA_X_DIGEST_SIZE != 384)
 230         mov     QWORD(tmp5), [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 3*SHA512_DIGEST_ROW_SIZE]
 231 %endif
 232         bswap   QWORD(tmp2)
 233         bswap   QWORD(tmp4)
 234         bswap   QWORD(tmp6)
 235 %if (SHA_X_DIGEST_SIZE != 384)
 236         bswap   QWORD(tmp5)
 237 %endif
 238         mov     [p + 0*8], QWORD(tmp2)
 239         mov     [p + 1*8], QWORD(tmp4)
 240         mov     [p + 2*8], QWORD(tmp6)
 241 %if (SHA_X_DIGEST_SIZE != 384)
 242         mov     [p + 3*8], QWORD(tmp5)
 243 %endif
 244         jmp     return
 245 copy_full_digest:
 246         ;; copy 32 bytes for SHA512 // 24 bytes for SHA384
 247         mov     QWORD(tmp2), [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 0*SHA512_DIGEST_ROW_SIZE]
 248         mov     QWORD(tmp4), [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 1*SHA512_DIGEST_ROW_SIZE]
 249         mov     QWORD(tmp6), [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 2*SHA512_DIGEST_ROW_SIZE]
 250         mov     QWORD(tmp5), [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 3*SHA512_DIGEST_ROW_SIZE]
 251         bswap   QWORD(tmp2)
 252         bswap   QWORD(tmp4)
 253         bswap   QWORD(tmp6)
 254         bswap   QWORD(tmp5)
 255         mov     [p + 0*8], QWORD(tmp2)
 256         mov     [p + 1*8], QWORD(tmp4)
 257         mov     [p + 2*8], QWORD(tmp6)
 258         mov     [p + 3*8], QWORD(tmp5)
 259
 260         mov     QWORD(tmp2), [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 4*SHA512_DIGEST_ROW_SIZE]
 261         mov     QWORD(tmp4), [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 5*SHA512_DIGEST_ROW_SIZE]
 262 %if (SHA_X_DIGEST_SIZE != 384)
 263         mov     QWORD(tmp6), [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 6*SHA512_DIGEST_ROW_SIZE]
 264         mov     QWORD(tmp5), [state + _args_digest_sha512 + SHA512_DIGEST_WORD_SIZE*idx + 7*SHA512_DIGEST_ROW_SIZE]
 265 %endif
 266         bswap   QWORD(tmp2)
 267         bswap   QWORD(tmp4)
 268 %if (SHA_X_DIGEST_SIZE != 384)
 269         bswap   QWORD(tmp6)
 270         bswap   QWORD(tmp5)
 271 %endif
 272         mov     [p + 4*8], QWORD(tmp2)
 273         mov     [p + 5*8], QWORD(tmp4)
 274 %if (SHA_X_DIGEST_SIZE != 384)
 275         mov     [p + 6*8], QWORD(tmp6)
 276         mov     [p + 7*8], QWORD(tmp5)
 277 %endif
 278
 279 return:
 280         mov     rbx, [rsp + _gpr_save + 8*0]
 281         mov     rbp, [rsp + _gpr_save + 8*1]
 282         mov     rsp, [rsp + _rsp_save]  ; original SP
 283         ret
 284
 285 %ifdef LINUX
 286 section .note.GNU-stack noalloc noexec nowrite progbits
 287 %endif