[ceph.git] / ceph / src / spdk / intel-ipsec-mb / sse / mb_mgr_aes_cmac_submit_flush_sse.asm

;;
;; Copyright (c) 2018, Intel Corporation
;;
;; Redistribution and use in source and binary forms, with or without
;; modification, are permitted provided that the following conditions are met:
;;
;;     * Redistributions of source code must retain the above copyright notice,
;;       this list of conditions and the following disclaimer.
;;     * Redistributions in binary form must reproduce the above copyright
;;       notice, this list of conditions and the following disclaimer in the
;;       documentation and/or other materials provided with the distribution.
;;     * Neither the name of Intel Corporation nor the names of its contributors
;;       may be used to endorse or promote products derived from this software
;;       without specific prior written permission.
;;
;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
;;


%include "include/os.asm"
%include "job_aes_hmac.asm"
%include "mb_mgr_datastruct.asm"

%include "include/reg_sizes.asm"
%include "include/memcpy.asm"
%include "include/const.inc"
;%define DO_DBGPRINT
%include "include/dbgprint.asm"

%ifndef AES128_CBC_MAC

%define AES128_CBC_MAC aes128_cbc_mac_x4
%define SUBMIT_JOB_AES_CMAC_AUTH submit_job_aes_cmac_auth_sse
%define FLUSH_JOB_AES_CMAC_AUTH flush_job_aes_cmac_auth_sse

%endif

extern AES128_CBC_MAC

section .data
default rel

align 16
len_masks:
	;ddq 0x0000000000000000000000000000FFFF
	dq 0x000000000000FFFF, 0x0000000000000000
	;ddq 0x000000000000000000000000FFFF0000
	dq 0x00000000FFFF0000, 0x0000000000000000
	;ddq 0x00000000000000000000FFFF00000000
	dq 0x0000FFFF00000000, 0x0000000000000000
	;ddq 0x0000000000000000FFFF000000000000
	dq 0xFFFF000000000000, 0x0000000000000000
one:	dq  1
two:	dq  2
three:	dq  3

section .text

%define APPEND(a,b) a %+ b

%ifdef LINUX
%define arg1	rdi
%define arg2	rsi
%else
%define arg1	rcx
%define arg2	rdx
%endif

%define state	arg1
%define job	arg2
%define len2	arg2

%define job_rax          rax

; idx needs to be in rbp
%define len              rbp
%define idx              rbp
%define tmp              rbp

%define lane             r8

%define iv               r9
%define m_last           r10
%define n                r11

%define unused_lanes     rbx
%define r                rbx

%define tmp3             r12
%define tmp4             r13
%define tmp2             r14

%define good_lane        r15
%define rbits            r15

; STACK_SPACE needs to be an odd multiple of 8
; This routine and its callee clobbers all GPRs
struc STACK
_gpr_save:	resq	8
_rsp_save:	resq	1
endstruc

;;; ===========================================================================
;;; ===========================================================================
;;; MACROS
;;; ===========================================================================
;;; ===========================================================================

;;; ===========================================================================
;;; AES CMAC job submit & flush
;;; ===========================================================================
;;; SUBMIT_FLUSH [in] - SUBMIT, FLUSH job selection
%macro GENERIC_SUBMIT_FLUSH_JOB_AES_CMAC_SSE 1
%define %%SUBMIT_FLUSH %1

        mov	rax, rsp
        sub	rsp, STACK_size
        and	rsp, -16

	mov	[rsp + _gpr_save + 8*0], rbx
	mov	[rsp + _gpr_save + 8*1], rbp
	mov	[rsp + _gpr_save + 8*2], r12
	mov	[rsp + _gpr_save + 8*3], r13
	mov	[rsp + _gpr_save + 8*4], r14
	mov	[rsp + _gpr_save + 8*5], r15
%ifndef LINUX
	mov	[rsp + _gpr_save + 8*6], rsi
	mov	[rsp + _gpr_save + 8*7], rdi
%endif
	mov	[rsp + _rsp_save], rax	; original SP

        ;; Find free lane
 	mov	unused_lanes, [state + _aes_cmac_unused_lanes]

%ifidn %%SUBMIT_FLUSH, SUBMIT

 	mov	lane, unused_lanes
        and	lane, 0xF
 	shr	unused_lanes, 4
 	mov	[state + _aes_cmac_unused_lanes], unused_lanes

        ;; Copy job info into lane
 	mov	[state + _aes_cmac_job_in_lane + lane*8], job
        ;; Copy keys into lane args
 	mov	tmp, [job + _key_expanded]
 	mov	[state + _aes_cmac_args_keys + lane*8], tmp
        mov     tmp, lane
        shl     tmp, 4  ; lane*16

        ;; Zero IV to store digest
        pxor    xmm0, xmm0
        movdqa  [state + _aes_cmac_args_IV + tmp], xmm0

        lea     m_last, [state + _aes_cmac_scratch + tmp]

        ;; calculate len
        ;; convert bits to bytes (message length in bits for CMAC)
        mov     len, [job + _msg_len_to_hash_in_bits]
        mov     rbits, len
        add     len, 7      ; inc len if there are remainder bits
        shr     len, 3
        and     rbits, 7

        ;; Check at least 1 or more blocks (get n)
        mov     n, len
        add     n, 0xf
        shr     n, 4

        ;; Check for partial block
        mov     r, len
        and     r, 0xf

        or      n, n   ; check one or more blocks?
        jz      %%_lt_one_block

        ;; One or more blocks, potentially partial
        mov     word [state + _aes_cmac_init_done + lane*2], 0

        mov     tmp2, [job + _src]
        add     tmp2, [job + _hash_start_src_offset_in_bytes]
        mov     [state + _aes_cmac_args_in + lane*8], tmp2

        ;; len = (n-1)*16
        lea     tmp2, [n - 1]
        shl     tmp2, 4
        movdqa  xmm0, [state + _aes_cmac_lens]
        XPINSRW xmm0, xmm1, tmp, lane, tmp2, scale_x16
        movdqa  [state + _aes_cmac_lens], xmm0

        ;; check remainder bits
        or      rbits, rbits
        jnz     %%_not_complete_block_3gpp

        ;; check if complete block
        or      r, r
        jz      %%_complete_block

%%_not_complete_block:
        ;; M_last = padding(M_n) XOR K2
        lea     tmp, [rel padding_0x80_tab16 + 16]
        sub     tmp, r
        movdqu  xmm0, [tmp]
        movdqa  [m_last], xmm0

        mov     tmp, [job + _src]
        add     tmp, [job + _hash_start_src_offset_in_bytes]
        lea     tmp3, [n - 1]
        shl     tmp3, 4
        add     tmp, tmp3

        memcpy_sse_16 m_last, tmp, r, tmp4, iv

        ;; src + n + r
        mov     tmp3, [job + _skey2]
        movdqa  xmm1, [m_last]
        movdqu  xmm0, [tmp3]
        pxor    xmm0, xmm1
        movdqa  [m_last], xmm0

%%_step_5:
        ;; Find min length
        movdqa  xmm0, [state + _aes_cmac_lens]
        phminposuw xmm1, xmm0

        cmp	byte [state + _aes_cmac_unused_lanes], 0xf
        jne	%%_return_null

%else ; end SUBMIT

        ;; Check at least one job
        bt      unused_lanes, 19
	jc      %%_return_null

      	;; Find a lane with a non-null job
	xor	good_lane, good_lane
	cmp	qword [state + _aes_cmac_job_in_lane + 1*8], 0
	cmovne	good_lane, [rel one]
	cmp	qword [state + _aes_cmac_job_in_lane + 2*8], 0
	cmovne	good_lane, [rel two]
	cmp	qword [state + _aes_cmac_job_in_lane + 3*8], 0
	cmovne	good_lane, [rel three]

	; Copy good_lane to empty lanes
	mov	tmp2, [state + _aes_cmac_args_in + good_lane*8]
	mov	tmp3, [state + _aes_cmac_args_keys + good_lane*8]
	shl	good_lane, 4 ; multiply by 16
	movdqa	xmm2, [state + _aes_cmac_args_IV + good_lane]
	movdqa	xmm0, [state + _aes_cmac_lens]

%assign I 0
%rep 4
	cmp	qword [state + _aes_cmac_job_in_lane + I*8], 0
	jne	APPEND(skip_,I)
	mov	[state + _aes_cmac_args_in + I*8], tmp2
	mov	[state + _aes_cmac_args_keys + I*8], tmp3
	movdqa	[state + _aes_cmac_args_IV + I*16], xmm2
	por	xmm0, [rel len_masks + 16*I]
APPEND(skip_,I):
%assign I (I+1)
%endrep
        ;; Find min length
        phminposuw xmm1, xmm0

%endif ; end FLUSH

%%_cmac_round:
	pextrw	len2, xmm1, 0	; min value
	pextrw	idx, xmm1, 1	; min index (0...3)
        cmp	len2, 0
	je	%%_len_is_0
        pshuflw	xmm1, xmm1, 0
	psubw	xmm0, xmm1
	movdqa	[state + _aes_cmac_lens], xmm0

        ; "state" and "args" are the same address, arg1
	; len2 is arg2
	call    AES128_CBC_MAC
	; state and idx are intact

        movdqa  xmm0, [state + _aes_cmac_lens]  ; preload lens
%%_len_is_0:
        ; Check if job complete
        test    word [state + _aes_cmac_init_done + idx*2], 0xffff
        jnz     %%_copy_complete_digest

        ; Finish step 6
        mov     word [state + _aes_cmac_init_done + idx*2], 1

        XPINSRW xmm0, xmm1, tmp3, idx, 16, scale_x16
        movdqa  [state + _aes_cmac_lens], xmm0

        phminposuw xmm1, xmm0 ; find min length

        mov     tmp3, idx
        shl     tmp3, 4  ; idx*16
        lea     m_last, [state + _aes_cmac_scratch + tmp3]
        mov     [state + _aes_cmac_args_in + idx*8], m_last

        jmp     %%_cmac_round

%%_copy_complete_digest:
        ; Job complete, copy digest to AT output
 	mov	job_rax, [state + _aes_cmac_job_in_lane + idx*8]

        mov     tmp4, idx
        shl     tmp4, 4
        lea     tmp3, [state + _aes_cmac_args_IV + tmp4]
        mov     tmp4, [job_rax + _auth_tag_output_len_in_bytes]
        mov     tmp2, [job_rax + _auth_tag_output]

        cmp     tmp4, 16
        jne     %%_ne_16_copy

        ;; 16 byte AT copy
        movdqu  xmm0, [tmp3]
        movdqu  [tmp2], xmm0
        jmp     %%_update_lanes

%%_ne_16_copy:
        memcpy_sse_16 tmp2, tmp3, tmp4, lane, iv

%%_update_lanes:
        ; Update unused lanes
        mov	unused_lanes, [state + _aes_cmac_unused_lanes]
        shl	unused_lanes, 4
 	or	unused_lanes, idx
 	mov	[state + _aes_cmac_unused_lanes], unused_lanes

        ; Set return job
        mov	job_rax, [state + _aes_cmac_job_in_lane + idx*8]

 	mov	qword [state + _aes_cmac_job_in_lane + idx*8], 0
 	or	dword [job_rax + _status], STS_COMPLETED_HMAC

%ifdef SAFE_DATA
        pxor    xmm0, xmm0
%ifidn %%SUBMIT_FLUSH, SUBMIT
        ;; Clear digest (in memory for IV) and scratch memory of returned job
        movdqa  [tmp3], xmm0

        shl     idx, 4
        movdqa  [state + _aes_cmac_scratch + idx], xmm0

%else
        ;; Clear digest and scratch memory of returned job and "NULL lanes"
%assign I 0
%rep 4
        cmp     qword [state + _aes_cmac_job_in_lane + I*8], 0
        jne     APPEND(skip_clear_,I)
        movdqa  [state + _aes_cmac_args_IV + I*16], xmm0
        movdqa  [state + _aes_cmac_scratch + I*16], xmm0
APPEND(skip_clear_,I):
%assign I (I+1)
%endrep
%endif ;; SUBMIT

%endif ;; SAFE_DATA

%%_return:
	mov	rbx, [rsp + _gpr_save + 8*0]
	mov	rbp, [rsp + _gpr_save + 8*1]
	mov	r12, [rsp + _gpr_save + 8*2]
	mov	r13, [rsp + _gpr_save + 8*3]
	mov	r14, [rsp + _gpr_save + 8*4]
	mov	r15, [rsp + _gpr_save + 8*5]
%ifndef LINUX
	mov	rsi, [rsp + _gpr_save + 8*6]
	mov	rdi, [rsp + _gpr_save + 8*7]
%endif
	mov	rsp, [rsp + _rsp_save]	; original SP
	ret

%%_return_null:
	xor	job_rax, job_rax
	jmp	%%_return

%ifidn %%SUBMIT_FLUSH, SUBMIT
%%_complete_block:

        ;; Block size aligned
        mov     tmp2, [job + _src]
        add     tmp2, [job + _hash_start_src_offset_in_bytes]
        lea     tmp3, [n - 1]
        shl     tmp3, 4
        add     tmp2, tmp3

        ;; M_last = M_n XOR K1
        mov     tmp3, [job + _skey1]
        movdqu  xmm0, [tmp3]
        movdqu  xmm1, [tmp2]
        pxor    xmm0, xmm1
        movdqa  [m_last], xmm0

        jmp     %%_step_5

%%_lt_one_block:
        ;; Single partial block
        mov     word [state + _aes_cmac_init_done + lane*2], 1
        mov     [state + _aes_cmac_args_in + lane*8], m_last

        movdqa  xmm0, [state + _aes_cmac_lens]
        XPINSRW xmm0, xmm1, tmp2, lane, 16, scale_x16
        movdqa  [state + _aes_cmac_lens], xmm0

        mov     n, 1
        jmp     %%_not_complete_block

%%_not_complete_block_3gpp:
        ;; bit pad last block
        ;; xor with skey2
        ;; copy to m_last

        ;; load pointer to src
        mov     tmp, [job + _src]
        add     tmp, [job + _hash_start_src_offset_in_bytes]
        lea     tmp3, [n - 1]
        shl     tmp3, 4
        add     tmp, tmp3

        ;; check if partial block
        or      r, r
        jz      %%_load_full_block_3gpp

        simd_load_sse_15_1 xmm0, tmp, r
        dec     r

%%_update_mlast_3gpp:
        ;; set last byte padding mask
        ;; shift into correct xmm idx

        ;; save and restore rcx on windows
%ifndef LINUX
	mov	tmp, rcx
%endif
        mov     rcx, rbits
        mov     tmp3, 0xff
        shr     tmp3, cl
        movq    xmm2, tmp3
        XPSLLB  xmm2, r, xmm1, tmp2

        ;; pad final byte
        pandn   xmm2, xmm0
%ifndef LINUX
	mov	rcx, tmp
%endif
        ;; set OR mask to pad final bit
        mov     tmp2, tmp3
        shr     tmp2, 1
        xor     tmp2, tmp3 ; XOR to get OR mask
        movq    xmm3, tmp2
        ;; xmm1 contains shift table from previous shift
        pshufb  xmm3, xmm1

        ;; load skey2 address
        mov     tmp3, [job + _skey2]
        movdqu  xmm1, [tmp3]

        ;; set final padding bit
        por     xmm2, xmm3

        ;; XOR last partial block with skey2
        ;; update mlast
        pxor    xmm2, xmm1
        movdqa  [m_last], xmm2

        jmp     %%_step_5

%%_load_full_block_3gpp:
        movdqu  xmm0, [tmp]
        mov     r, 0xf
        jmp     %%_update_mlast_3gpp
%endif
%endmacro


align 64
; JOB_AES_HMAC * submit_job_aes_cmac_auth_sse(MB_MGR_CMAC_OOO *state, JOB_AES_HMAC *job)
; arg 1 : state
; arg 2 : job
MKGLOBAL(SUBMIT_JOB_AES_CMAC_AUTH,function,internal)
SUBMIT_JOB_AES_CMAC_AUTH:
        GENERIC_SUBMIT_FLUSH_JOB_AES_CMAC_SSE SUBMIT

; JOB_AES_HMAC * flush_job_aes_cmac_auth_sse(MB_MGR_CMAC_OOO *state)
; arg 1 : state
MKGLOBAL(FLUSH_JOB_AES_CMAC_AUTH,function,internal)
FLUSH_JOB_AES_CMAC_AUTH:
        GENERIC_SUBMIT_FLUSH_JOB_AES_CMAC_SSE FLUSH


%ifdef LINUX
section .note.GNU-stack noalloc noexec nowrite progbits
%endif
Commit	Line	Data
9f95a23c TL	1	;;
	2	;; Copyright (c) 2018, Intel Corporation
	3	;;
	4	;; Redistribution and use in source and binary forms, with or without
	5	;; modification, are permitted provided that the following conditions are met:
	6	;;
	7	;; * Redistributions of source code must retain the above copyright notice,
	8	;; this list of conditions and the following disclaimer.
	9	;; * Redistributions in binary form must reproduce the above copyright
	10	;; notice, this list of conditions and the following disclaimer in the
	11	;; documentation and/or other materials provided with the distribution.
	12	;; * Neither the name of Intel Corporation nor the names of its contributors
	13	;; may be used to endorse or promote products derived from this software
	14	;; without specific prior written permission.
	15	;;
	16	;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
	17	;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	18	;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
	19	;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
	20	;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
	21	;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
	22	;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
	23	;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
	24	;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	25	;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	26	;;
	27
	28
f67539c2	29	%include "include/os.asm"
9f95a23c TL	30	%include "job_aes_hmac.asm"
	31	%include "mb_mgr_datastruct.asm"
	32
f67539c2 TL	33	%include "include/reg_sizes.asm"
	34	%include "include/memcpy.asm"
	35	%include "include/const.inc"
9f95a23c	36	;%define DO_DBGPRINT
f67539c2	37	%include "include/dbgprint.asm"
9f95a23c TL	38
	39	%ifndef AES128_CBC_MAC
	40
	41	%define AES128_CBC_MAC aes128_cbc_mac_x4
	42	%define SUBMIT_JOB_AES_CMAC_AUTH submit_job_aes_cmac_auth_sse
	43	%define FLUSH_JOB_AES_CMAC_AUTH flush_job_aes_cmac_auth_sse
	44
	45	%endif
	46
	47	extern AES128_CBC_MAC
	48
	49	section .data
	50	default rel
	51
	52	align 16
	53	len_masks:
	54	;ddq 0x0000000000000000000000000000FFFF
	55	dq 0x000000000000FFFF, 0x0000000000000000
	56	;ddq 0x000000000000000000000000FFFF0000
	57	dq 0x00000000FFFF0000, 0x0000000000000000
	58	;ddq 0x00000000000000000000FFFF00000000
	59	dq 0x0000FFFF00000000, 0x0000000000000000
	60	;ddq 0x0000000000000000FFFF000000000000
	61	dq 0xFFFF000000000000, 0x0000000000000000
	62	one: dq 1
	63	two: dq 2
	64	three: dq 3
	65
	66	section .text
	67
	68	%define APPEND(a,b) a %+ b
	69
	70	%ifdef LINUX
	71	%define arg1 rdi
	72	%define arg2 rsi
	73	%else
	74	%define arg1 rcx
	75	%define arg2 rdx
	76	%endif
	77
	78	%define state arg1
	79	%define job arg2
	80	%define len2 arg2
	81
	82	%define job_rax rax
	83
	84	; idx needs to be in rbp
	85	%define len rbp
	86	%define idx rbp
	87	%define tmp rbp
	88
	89	%define lane r8
	90
	91	%define iv r9
	92	%define m_last r10
	93	%define n r11
	94
	95	%define unused_lanes rbx
	96	%define r rbx
	97
	98	%define tmp3 r12
	99	%define tmp4 r13
	100	%define tmp2 r14
	101
9f95a23c	102	%define good_lane r15
f67539c2	103	%define rbits r15
9f95a23c TL	104
	105	; STACK_SPACE needs to be an odd multiple of 8
	106	; This routine and its callee clobbers all GPRs
	107	struc STACK
	108	_gpr_save: resq 8
	109	_rsp_save: resq 1
	110	endstruc
	111
	112	;;; ===========================================================================
	113	;;; ===========================================================================
	114	;;; MACROS
	115	;;; ===========================================================================
	116	;;; ===========================================================================
	117
	118	;;; ===========================================================================
	119	;;; AES CMAC job submit & flush
	120	;;; ===========================================================================
	121	;;; SUBMIT_FLUSH [in] - SUBMIT, FLUSH job selection
	122	%macro GENERIC_SUBMIT_FLUSH_JOB_AES_CMAC_SSE 1
	123	%define %%SUBMIT_FLUSH %1
	124
	125	mov rax, rsp
	126	sub rsp, STACK_size
	127	and rsp, -16
	128
	129	mov [rsp + _gpr_save + 8*0], rbx
	130	mov [rsp + _gpr_save + 8*1], rbp
	131	mov [rsp + _gpr_save + 8*2], r12
	132	mov [rsp + _gpr_save + 8*3], r13
	133	mov [rsp + _gpr_save + 8*4], r14
	134	mov [rsp + _gpr_save + 8*5], r15
	135	%ifndef LINUX
	136	mov [rsp + _gpr_save + 8*6], rsi
	137	mov [rsp + _gpr_save + 8*7], rdi
	138	%endif
	139	mov [rsp + _rsp_save], rax ; original SP
	140
	141	;; Find free lane
	142	mov unused_lanes, [state + _aes_cmac_unused_lanes]
	143
	144	%ifidn %%SUBMIT_FLUSH, SUBMIT
9f95a23c TL	145
	146	mov lane, unused_lanes
	147	and lane, 0xF
	148	shr unused_lanes, 4
	149	mov [state + _aes_cmac_unused_lanes], unused_lanes
	150
	151	;; Copy job info into lane
	152	mov [state + _aes_cmac_job_in_lane + lane*8], job
	153	;; Copy keys into lane args
	154	mov tmp, [job + _key_expanded]
	155	mov [state + _aes_cmac_args_keys + lane*8], tmp
	156	mov tmp, lane
	157	shl tmp, 4 ; lane*16
	158
	159	;; Zero IV to store digest
	160	pxor xmm0, xmm0
	161	movdqa [state + _aes_cmac_args_IV + tmp], xmm0
	162
	163	lea m_last, [state + _aes_cmac_scratch + tmp]
	164
f67539c2 TL	165	;; calculate len
	166	;; convert bits to bytes (message length in bits for CMAC)
	167	mov len, [job + _msg_len_to_hash_in_bits]
	168	mov rbits, len
	169	add len, 7 ; inc len if there are remainder bits
	170	shr len, 3
	171	and rbits, 7
	172
9f95a23c	173	;; Check at least 1 or more blocks (get n)
9f95a23c TL	174	mov n, len
	175	add n, 0xf
	176	shr n, 4
	177
	178	;; Check for partial block
	179	mov r, len
	180	and r, 0xf
	181
	182	or n, n ; check one or more blocks?
	183	jz %%_lt_one_block
	184
	185	;; One or more blocks, potentially partial
	186	mov word [state + _aes_cmac_init_done + lane*2], 0
	187
	188	mov tmp2, [job + _src]
	189	add tmp2, [job + _hash_start_src_offset_in_bytes]
	190	mov [state + _aes_cmac_args_in + lane*8], tmp2
	191
	192	;; len = (n-1)*16
	193	lea tmp2, [n - 1]
	194	shl tmp2, 4
	195	movdqa xmm0, [state + _aes_cmac_lens]
	196	XPINSRW xmm0, xmm1, tmp, lane, tmp2, scale_x16
	197	movdqa [state + _aes_cmac_lens], xmm0
	198
f67539c2 TL	199	;; check remainder bits
	200	or rbits, rbits
	201	jnz %%_not_complete_block_3gpp
	202
	203	;; check if complete block
9f95a23c TL	204	or r, r
	205	jz %%_complete_block
	206
	207	%%_not_complete_block:
	208	;; M_last = padding(M_n) XOR K2
	209	lea tmp, [rel padding_0x80_tab16 + 16]
	210	sub tmp, r
	211	movdqu xmm0, [tmp]
	212	movdqa [m_last], xmm0
	213
	214	mov tmp, [job + _src]
	215	add tmp, [job + _hash_start_src_offset_in_bytes]
	216	lea tmp3, [n - 1]
	217	shl tmp3, 4
	218	add tmp, tmp3
	219
	220	memcpy_sse_16 m_last, tmp, r, tmp4, iv
	221
	222	;; src + n + r
	223	mov tmp3, [job + _skey2]
	224	movdqa xmm1, [m_last]
	225	movdqu xmm0, [tmp3]
	226	pxor xmm0, xmm1
	227	movdqa [m_last], xmm0
	228
	229	%%_step_5:
	230	;; Find min length
	231	movdqa xmm0, [state + _aes_cmac_lens]
	232	phminposuw xmm1, xmm0
	233
	234	cmp byte [state + _aes_cmac_unused_lanes], 0xf
	235	jne %%_return_null
	236
	237	%else ; end SUBMIT
	238
	239	;; Check at least one job
	240	bt unused_lanes, 19
	241	jc %%_return_null
	242
	243	;; Find a lane with a non-null job
	244	xor good_lane, good_lane
	245	cmp qword [state + _aes_cmac_job_in_lane + 1*8], 0
	246	cmovne good_lane, [rel one]
	247	cmp qword [state + _aes_cmac_job_in_lane + 2*8], 0
	248	cmovne good_lane, [rel two]
	249	cmp qword [state + _aes_cmac_job_in_lane + 3*8], 0
	250	cmovne good_lane, [rel three]
	251
	252	; Copy good_lane to empty lanes
	253	mov tmp2, [state + _aes_cmac_args_in + good_lane*8]
	254	mov tmp3, [state + _aes_cmac_args_keys + good_lane*8]
	255	shl good_lane, 4 ; multiply by 16
	256	movdqa xmm2, [state + _aes_cmac_args_IV + good_lane]
	257	movdqa xmm0, [state + _aes_cmac_lens]
	258
	259	%assign I 0
	260	%rep 4
	261	cmp qword [state + _aes_cmac_job_in_lane + I*8], 0
	262	jne APPEND(skip_,I)
	263	mov [state + _aes_cmac_args_in + I*8], tmp2
	264	mov [state + _aes_cmac_args_keys + I*8], tmp3
	265	movdqa [state + _aes_cmac_args_IV + I*16], xmm2
	266	por xmm0, [rel len_masks + 16*I]
	267	APPEND(skip_,I):
268	%assign I (I+1)
269	%endrep
270	;; Find min length
271	phminposuw xmm1, xmm0
272
273	%endif ; end FLUSH
274
275	%%_cmac_round:
276	pextrw len2, xmm1, 0 ; min value
277	pextrw idx, xmm1, 1 ; min index (0...3)
278	cmp len2, 0
279	je %%_len_is_0
280	pshuflw xmm1, xmm1, 0
281	psubw xmm0, xmm1
282	movdqa [state + _aes_cmac_lens], xmm0
283
284	; "state" and "args" are the same address, arg1
285	; len2 is arg2
286	call AES128_CBC_MAC
287	; state and idx are intact
288
f67539c2	289	movdqa xmm0, [state + _aes_cmac_lens] ; preload lens
9f95a23c TL	290	%%_len_is_0:
	291	; Check if job complete
	292	test word [state + _aes_cmac_init_done + idx*2], 0xffff
	293	jnz %%_copy_complete_digest
	294
	295	; Finish step 6
	296	mov word [state + _aes_cmac_init_done + idx*2], 1
	297
9f95a23c TL	298	XPINSRW xmm0, xmm1, tmp3, idx, 16, scale_x16
	299	movdqa [state + _aes_cmac_lens], xmm0
	300
	301	phminposuw xmm1, xmm0 ; find min length
	302
	303	mov tmp3, idx
	304	shl tmp3, 4 ; idx*16
	305	lea m_last, [state + _aes_cmac_scratch + tmp3]
	306	mov [state + _aes_cmac_args_in + idx*8], m_last
	307
	308	jmp %%_cmac_round
	309
	310	%%_copy_complete_digest:
	311	; Job complete, copy digest to AT output
	312	mov job_rax, [state + _aes_cmac_job_in_lane + idx*8]
	313
	314	mov tmp4, idx
	315	shl tmp4, 4
	316	lea tmp3, [state + _aes_cmac_args_IV + tmp4]
	317	mov tmp4, [job_rax + _auth_tag_output_len_in_bytes]
	318	mov tmp2, [job_rax + _auth_tag_output]
	319
	320	cmp tmp4, 16
	321	jne %%_ne_16_copy
	322
	323	;; 16 byte AT copy
	324	movdqu xmm0, [tmp3]
	325	movdqu [tmp2], xmm0
	326	jmp %%_update_lanes
	327
	328	%%_ne_16_copy:
	329	memcpy_sse_16 tmp2, tmp3, tmp4, lane, iv
	330
	331	%%_update_lanes:
	332	; Update unused lanes
	333	mov unused_lanes, [state + _aes_cmac_unused_lanes]
	334	shl unused_lanes, 4
	335	or unused_lanes, idx
	336	mov [state + _aes_cmac_unused_lanes], unused_lanes
	337
	338	; Set return job
	339	mov job_rax, [state + _aes_cmac_job_in_lane + idx*8]
	340
	341	mov qword [state + _aes_cmac_job_in_lane + idx*8], 0
	342	or dword [job_rax + _status], STS_COMPLETED_HMAC
	343
f67539c2 TL	344	%ifdef SAFE_DATA
	345	pxor xmm0, xmm0
	346	%ifidn %%SUBMIT_FLUSH, SUBMIT
	347	;; Clear digest (in memory for IV) and scratch memory of returned job
	348	movdqa [tmp3], xmm0
	349
	350	shl idx, 4
	351	movdqa [state + _aes_cmac_scratch + idx], xmm0
	352
	353	%else
	354	;; Clear digest and scratch memory of returned job and "NULL lanes"
	355	%assign I 0
	356	%rep 4
	357	cmp qword [state + _aes_cmac_job_in_lane + I*8], 0
	358	jne APPEND(skip_clear_,I)
	359	movdqa [state + _aes_cmac_args_IV + I*16], xmm0
	360	movdqa [state + _aes_cmac_scratch + I*16], xmm0
	361	APPEND(skip_clear_,I):
	362	%assign I (I+1)
	363	%endrep
	364	%endif ;; SUBMIT
	365
	366	%endif ;; SAFE_DATA
	367
9f95a23c TL	368	%%_return:
	369	mov rbx, [rsp + _gpr_save + 8*0]
	370	mov rbp, [rsp + _gpr_save + 8*1]
	371	mov r12, [rsp + _gpr_save + 8*2]
	372	mov r13, [rsp + _gpr_save + 8*3]
	373	mov r14, [rsp + _gpr_save + 8*4]
	374	mov r15, [rsp + _gpr_save + 8*5]
	375	%ifndef LINUX
	376	mov rsi, [rsp + _gpr_save + 8*6]
	377	mov rdi, [rsp + _gpr_save + 8*7]
	378	%endif
	379	mov rsp, [rsp + _rsp_save] ; original SP
	380	ret
	381
	382	%%_return_null:
	383	xor job_rax, job_rax
	384	jmp %%_return
	385
	386	%ifidn %%SUBMIT_FLUSH, SUBMIT
	387	%%_complete_block:
9f95a23c TL	388
	389	;; Block size aligned
	390	mov tmp2, [job + _src]
	391	add tmp2, [job + _hash_start_src_offset_in_bytes]
	392	lea tmp3, [n - 1]
	393	shl tmp3, 4
	394	add tmp2, tmp3
	395
	396	;; M_last = M_n XOR K1
	397	mov tmp3, [job + _skey1]
	398	movdqu xmm0, [tmp3]
	399	movdqu xmm1, [tmp2]
	400	pxor xmm0, xmm1
	401	movdqa [m_last], xmm0
	402
	403	jmp %%_step_5
	404
	405	%%_lt_one_block:
	406	;; Single partial block
	407	mov word [state + _aes_cmac_init_done + lane*2], 1
	408	mov [state + _aes_cmac_args_in + lane*8], m_last
	409
	410	movdqa xmm0, [state + _aes_cmac_lens]
	411	XPINSRW xmm0, xmm1, tmp2, lane, 16, scale_x16
	412	movdqa [state + _aes_cmac_lens], xmm0
	413
	414	mov n, 1
	415	jmp %%_not_complete_block
f67539c2 TL	416
	417	%%_not_complete_block_3gpp:
	418	;; bit pad last block
	419	;; xor with skey2
	420	;; copy to m_last
	421
	422	;; load pointer to src
	423	mov tmp, [job + _src]
	424	add tmp, [job + _hash_start_src_offset_in_bytes]
	425	lea tmp3, [n - 1]
	426	shl tmp3, 4
	427	add tmp, tmp3
	428
	429	;; check if partial block
	430	or r, r
	431	jz %%_load_full_block_3gpp
	432
	433	simd_load_sse_15_1 xmm0, tmp, r
	434	dec r
	435
	436	%%_update_mlast_3gpp:
	437	;; set last byte padding mask
	438	;; shift into correct xmm idx
	439
	440	;; save and restore rcx on windows
	441	%ifndef LINUX
	442	mov tmp, rcx
	443	%endif
	444	mov rcx, rbits
	445	mov tmp3, 0xff
	446	shr tmp3, cl
	447	movq xmm2, tmp3
	448	XPSLLB xmm2, r, xmm1, tmp2
	449
	450	;; pad final byte
	451	pandn xmm2, xmm0
	452	%ifndef LINUX
	453	mov rcx, tmp
	454	%endif
	455	;; set OR mask to pad final bit
	456	mov tmp2, tmp3
	457	shr tmp2, 1
	458	xor tmp2, tmp3 ; XOR to get OR mask
	459	movq xmm3, tmp2
	460	;; xmm1 contains shift table from previous shift
	461	pshufb xmm3, xmm1
	462
	463	;; load skey2 address
	464	mov tmp3, [job + _skey2]
	465	movdqu xmm1, [tmp3]
	466
	467	;; set final padding bit
	468	por xmm2, xmm3
	469
	470	;; XOR last partial block with skey2
	471	;; update mlast
	472	pxor xmm2, xmm1
	473	movdqa [m_last], xmm2
	474
	475	jmp %%_step_5
	476
	477	%%_load_full_block_3gpp:
	478	movdqu xmm0, [tmp]
	479	mov r, 0xf
480	jmp %%_update_mlast_3gpp
9f95a23c TL	481	%endif
	482	%endmacro
	483
	484
	485	align 64
f67539c2	486	; JOB_AES_HMAC * submit_job_aes_cmac_auth_sse(MB_MGR_CMAC_OOO state, JOB_AES_HMAC job)
9f95a23c TL	487	; arg 1 : state
	488	; arg 2 : job
	489	MKGLOBAL(SUBMIT_JOB_AES_CMAC_AUTH,function,internal)
	490	SUBMIT_JOB_AES_CMAC_AUTH:
	491	GENERIC_SUBMIT_FLUSH_JOB_AES_CMAC_SSE SUBMIT
	492
f67539c2	493	; JOB_AES_HMAC * flush_job_aes_cmac_auth_sse(MB_MGR_CMAC_OOO *state)
9f95a23c TL	494	; arg 1 : state
	495	MKGLOBAL(FLUSH_JOB_AES_CMAC_AUTH,function,internal)
	496	FLUSH_JOB_AES_CMAC_AUTH:
	497	GENERIC_SUBMIT_FLUSH_JOB_AES_CMAC_SSE FLUSH
	498
	499
	500	%ifdef LINUX
	501	section .note.GNU-stack noalloc noexec nowrite progbits
	502	%endif