ceph/src/spdk/intel-ipsec-mb/avx/mb_mgr_aes_xcbc_flush_avx.asm

   1 ;;
   2 ;; Copyright (c) 2012-2018, Intel Corporation
   3 ;;
   4 ;; Redistribution and use in source and binary forms, with or without
   5 ;; modification, are permitted provided that the following conditions are met:
   6 ;;
   7 ;;     * Redistributions of source code must retain the above copyright notice,
   8 ;;       this list of conditions and the following disclaimer.
   9 ;;     * Redistributions in binary form must reproduce the above copyright
  10 ;;       notice, this list of conditions and the following disclaimer in the
  11 ;;       documentation and/or other materials provided with the distribution.
  12 ;;     * Neither the name of Intel Corporation nor the names of its contributors
  13 ;;       may be used to endorse or promote products derived from this software
  14 ;;       without specific prior written permission.
  15 ;;
  16 ;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  17 ;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18 ;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  19 ;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
  20 ;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21 ;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
  22 ;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
  23 ;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  24 ;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  25 ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  26 ;;
  27
  28 %include "include/os.asm"
  29 %include "job_aes_hmac.asm"
  30 %include "mb_mgr_datastruct.asm"
  31
  32 %include "include/reg_sizes.asm"
  33
  34 %ifndef AES_XCBC_X8
  35 %define AES_XCBC_X8 aes_xcbc_mac_128_x8
  36 %define FLUSH_JOB_AES_XCBC flush_job_aes_xcbc_avx
  37 %endif
  38
  39 ; void AES_XCBC_X8(AES_XCBC_ARGS_x8 *args, UINT64 len_in_bytes);
  40 extern AES_XCBC_X8
  41
  42 section .data
  43 default rel
  44
  45 align 16
  46 len_masks:
  47         ;ddq 0x0000000000000000000000000000FFFF
  48         dq 0x000000000000FFFF, 0x0000000000000000
  49         ;ddq 0x000000000000000000000000FFFF0000
  50         dq 0x00000000FFFF0000, 0x0000000000000000
  51         ;ddq 0x00000000000000000000FFFF00000000
  52         dq 0x0000FFFF00000000, 0x0000000000000000
  53         ;ddq 0x0000000000000000FFFF000000000000
  54         dq 0xFFFF000000000000, 0x0000000000000000
  55         ;ddq 0x000000000000FFFF0000000000000000
  56         dq 0x0000000000000000, 0x000000000000FFFF
  57         ;ddq 0x00000000FFFF00000000000000000000
  58         dq 0x0000000000000000, 0x00000000FFFF0000
  59         ;ddq 0x0000FFFF000000000000000000000000
  60         dq 0x0000000000000000, 0x0000FFFF00000000
  61         ;ddq 0xFFFF0000000000000000000000000000
  62         dq 0x0000000000000000, 0xFFFF000000000000
  63 dupw:
  64         ;ddq 0x01000100010001000100010001000100
  65         dq 0x0100010001000100, 0x0100010001000100
  66 one:    dq  1
  67 two:    dq  2
  68 three:  dq  3
  69 four:   dq  4
  70 five:   dq  5
  71 six:    dq  6
  72 seven:  dq  7
  73
  74 section .text
  75
  76 %define APPEND(a,b) a %+ b
  77
  78 %ifdef LINUX
  79 %define arg1    rdi
  80 %define arg2    rsi
  81 %else
  82 %define arg1    rcx
  83 %define arg2    rdx
  84 %endif
  85
  86 %define state   arg1
  87 %define job     arg2
  88 %define len2    arg2
  89
  90 %define job_rax          rax
  91
  92 %if 1
  93 %define unused_lanes     rbx
  94 %define tmp1             rbx
  95
  96 %define icv              rdx
  97
  98 %define tmp2             rax
  99
 100 ; idx needs to be in rbp
 101 %define tmp              r10
 102 %define idx              rbp
 103
 104 %define tmp3             r8
 105 %define lane_data        r9
 106 %endif
 107
 108 ; STACK_SPACE needs to be an odd multiple of 8
 109 ; This routine and its callee clobbers all GPRs
 110 struc STACK
 111 _gpr_save:      resq    8
 112 _rsp_save:      resq    1
 113 endstruc
 114
 115 ; JOB* FLUSH_JOB_AES_XCBC(MB_MGR_AES_XCBC_OOO *state, JOB_AES_HMAC *job)
 116 ; arg 1 : state
 117 ; arg 2 : job
 118 MKGLOBAL(FLUSH_JOB_AES_XCBC,function,internal)
 119 FLUSH_JOB_AES_XCBC:
 120
 121         mov     rax, rsp
 122         sub     rsp, STACK_size
 123         and     rsp, -16
 124
 125         mov     [rsp + _gpr_save + 8*0], rbx
 126         mov     [rsp + _gpr_save + 8*1], rbp
 127         mov     [rsp + _gpr_save + 8*2], r12
 128         mov     [rsp + _gpr_save + 8*3], r13
 129         mov     [rsp + _gpr_save + 8*4], r14
 130         mov     [rsp + _gpr_save + 8*5], r15
 131 %ifndef LINUX
 132         mov     [rsp + _gpr_save + 8*6], rsi
 133         mov     [rsp + _gpr_save + 8*7], rdi
 134 %endif
 135         mov     [rsp + _rsp_save], rax  ; original SP
 136
 137         ; check for empty
 138         mov     unused_lanes, [state + _aes_xcbc_unused_lanes]
 139         bt      unused_lanes, 32+3
 140         jc      return_null
 141
 142         ; find a lane with a non-null job
 143         xor     idx, idx
 144         cmp     qword [state + _aes_xcbc_ldata + 1 * _XCBC_LANE_DATA_size + _xcbc_job_in_lane], 0
 145         cmovne  idx, [rel one]
 146         cmp     qword [state + _aes_xcbc_ldata + 2 * _XCBC_LANE_DATA_size + _xcbc_job_in_lane], 0
 147         cmovne  idx, [rel two]
 148         cmp     qword [state + _aes_xcbc_ldata + 3 * _XCBC_LANE_DATA_size + _xcbc_job_in_lane], 0
 149         cmovne  idx, [rel three]
 150         cmp     qword [state + _aes_xcbc_ldata + 4 * _XCBC_LANE_DATA_size + _xcbc_job_in_lane], 0
 151         cmovne  idx, [rel four]
 152         cmp     qword [state + _aes_xcbc_ldata + 5 * _XCBC_LANE_DATA_size + _xcbc_job_in_lane], 0
 153         cmovne  idx, [rel five]
 154         cmp     qword [state + _aes_xcbc_ldata + 6 * _XCBC_LANE_DATA_size + _xcbc_job_in_lane], 0
 155         cmovne  idx, [rel six]
 156         cmp     qword [state + _aes_xcbc_ldata + 7 * _XCBC_LANE_DATA_size + _xcbc_job_in_lane], 0
 157         cmovne  idx, [rel seven]
 158
 159 copy_lane_data:
 160         ; copy idx to empty lanes
 161         mov     tmp1, [state + _aes_xcbc_args_in + idx*8]
 162         mov     tmp3, [state + _aes_xcbc_args_keys + idx*8]
 163         shl     idx, 4 ; multiply by 16
 164         vmovdqa xmm2, [state + _aes_xcbc_args_ICV + idx]
 165         vmovdqa xmm0, [state + _aes_xcbc_lens]
 166
 167 %assign I 0
 168 %rep 8
 169         cmp     qword [state + _aes_xcbc_ldata + I * _XCBC_LANE_DATA_size + _xcbc_job_in_lane], 0
 170         jne     APPEND(skip_,I)
 171         mov     [state + _aes_xcbc_args_in + I*8], tmp1
 172         mov     [state + _aes_xcbc_args_keys + I*8], tmp3
 173         vmovdqa [state + _aes_xcbc_args_ICV + I*16], xmm2
 174         vpor    xmm0, xmm0, [rel len_masks + 16*I]
 175 APPEND(skip_,I):
 176 %assign I (I+1)
 177 %endrep
 178
 179         vmovdqa [state + _aes_xcbc_lens], xmm0
 180
 181         ; Find min length
 182         vphminposuw     xmm1, xmm0
 183         vpextrw DWORD(len2), xmm1, 0    ; min value
 184         vpextrw DWORD(idx), xmm1, 1     ; min index (0...3)
 185         cmp     len2, 0
 186         je      len_is_0
 187
 188         vpshufb xmm1, xmm1, [rel dupw]   ; duplicate words across all lanes
 189         vpsubw  xmm0, xmm0, xmm1
 190         vmovdqa [state + _aes_xcbc_lens], xmm0
 191
 192         ; "state" and "args" are the same address, arg1
 193         ; len is arg2
 194         call    AES_XCBC_X8
 195         ; state and idx are intact
 196
 197 len_is_0:
 198         ; process completed job "idx"
 199         imul    lane_data, idx, _XCBC_LANE_DATA_size
 200         lea     lane_data, [state + _aes_xcbc_ldata + lane_data]
 201         cmp     dword [lane_data + _xcbc_final_done], 0
 202         jne     end_loop
 203
 204         mov     dword [lane_data + _xcbc_final_done], 1
 205         mov     word [state + _aes_xcbc_lens + 2*idx], 16
 206         lea     tmp, [lane_data + _xcbc_final_block]
 207         mov     [state + _aes_xcbc_args_in + 8*idx], tmp
 208         jmp     copy_lane_data
 209
 210 end_loop:
 211         mov     job_rax, [lane_data + _xcbc_job_in_lane]
 212         mov     icv, [job_rax + _auth_tag_output]
 213         mov     unused_lanes, [state + _aes_xcbc_unused_lanes]
 214         mov     qword [lane_data + _xcbc_job_in_lane], 0
 215         or      dword [job_rax + _status], STS_COMPLETED_HMAC
 216         shl     unused_lanes, 4
 217         or      unused_lanes, idx
 218         shl     idx, 4 ; multiply by 16
 219         mov     [state + _aes_xcbc_unused_lanes], unused_lanes
 220
 221         ; copy 12 bytes
 222         vmovdqa xmm0, [state + _aes_xcbc_args_ICV + idx]
 223         vmovq   [icv], xmm0
 224         vpextrd [icv + 8], xmm0, 2
 225
 226 %ifdef SAFE_DATA
 227         vpxor   xmm0, xmm0
 228         ;; Clear ICV's and final blocks in returned job and NULL lanes
 229 %assign I 0
 230 %rep 8
 231         cmp     qword [state + _aes_xcbc_ldata + I * _XCBC_LANE_DATA_size + _xcbc_job_in_lane], 0
 232         jne     APPEND(skip_clear_,I)
 233         vmovdqa [state + _aes_xcbc_args_ICV + I*16], xmm0
 234         lea     lane_data, [state + _aes_xcbc_ldata + (I * _XCBC_LANE_DATA_size)]
 235         vmovdqa [lane_data + _xcbc_final_block], xmm0
 236         vmovdqa [lane_data + _xcbc_final_block + 16], xmm0
 237 APPEND(skip_clear_,I):
 238 %assign I (I+1)
 239 %endrep
 240 %endif
 241
 242 return:
 243
 244         mov     rbx, [rsp + _gpr_save + 8*0]
 245         mov     rbp, [rsp + _gpr_save + 8*1]
 246         mov     r12, [rsp + _gpr_save + 8*2]
 247         mov     r13, [rsp + _gpr_save + 8*3]
 248         mov     r14, [rsp + _gpr_save + 8*4]
 249         mov     r15, [rsp + _gpr_save + 8*5]
 250 %ifndef LINUX
 251         mov     rsi, [rsp + _gpr_save + 8*6]
 252         mov     rdi, [rsp + _gpr_save + 8*7]
 253 %endif
 254         mov     rsp, [rsp + _rsp_save]  ; original SP
 255
 256         ret
 257
 258 return_null:
 259         xor     job_rax, job_rax
 260         jmp     return
 261
 262 %ifdef LINUX
 263 section .note.GNU-stack noalloc noexec nowrite progbits
 264 %endif