[ceph.git] / ceph / src / crypto / isa-l / isa-l_crypto / md5_mb / md5_mb_mgr_flush_sse.asm

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;  Copyright(c) 2011-2016 Intel Corporation All rights reserved.
;
;  Redistribution and use in source and binary forms, with or without
;  modification, are permitted provided that the following conditions
;  are met:
;    * Redistributions of source code must retain the above copyright
;      notice, this list of conditions and the following disclaimer.
;    * Redistributions in binary form must reproduce the above copyright
;      notice, this list of conditions and the following disclaimer in
;      the documentation and/or other materials provided with the
;      distribution.
;    * Neither the name of Intel Corporation nor the names of its
;      contributors may be used to endorse or promote products derived
;      from this software without specific prior written permission.
;
;  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
;  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
;  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
;  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
;  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
;  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
;  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
;  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
;  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
;  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
;  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

%include "md5_job.asm"
%include "md5_mb_mgr_datastruct.asm"

%include "reg_sizes.asm"

extern md5_mb_x4x2_sse

[bits 64]
default rel
section .text

%if 1
%ifidn __OUTPUT_FORMAT__, elf64
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
; UN*X register definitions
%define arg1    rdi ; rcx
%define arg2    rsi ; rdx

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

%else

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
; WINDOWS register definitions
%define arg1    rcx
%define arg2    rdx

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
%endif

; Common register definitions

%define state   arg1
%define len2    arg2

; idx must be a register not clobberred by md5_mb_x4x2_sse
%define idx             r8

%define unused_lanes    r9

%define lane_data       r10

%define job_rax         rax
%define tmp             rax

%endif ;; if 1

; STACK_SPACE needs to be an odd multiple of 8
_XMM_SAVE_SIZE  equ 10*16
_GPR_SAVE_SIZE  equ 8*8
_ALIGN_SIZE     equ 8

_XMM_SAVE       equ 0
_GPR_SAVE       equ _XMM_SAVE + _XMM_SAVE_SIZE
STACK_SPACE     equ _GPR_SAVE + _GPR_SAVE_SIZE + _ALIGN_SIZE

%define APPEND(a,b) a %+ b

; JOB* md5_mb_mgr_flush_sse(MB_MGR_HMAC_OOO *state)
; arg 1 : rcx : state
mk_global md5_mb_mgr_flush_sse, function
md5_mb_mgr_flush_sse:
	endbranch
        sub     rsp, STACK_SPACE
        mov     [rsp + _GPR_SAVE + 8*0], rbx
        mov     [rsp + _GPR_SAVE + 8*3], rbp
        mov     [rsp + _GPR_SAVE + 8*4], r12
        mov     [rsp + _GPR_SAVE + 8*5], r13
        mov     [rsp + _GPR_SAVE + 8*6], r14
        mov     [rsp + _GPR_SAVE + 8*7], r15
%ifidn __OUTPUT_FORMAT__, win64
        mov     [rsp + _GPR_SAVE + 8*1], rsi
        mov     [rsp + _GPR_SAVE + 8*2], rdi
        movdqa  [rsp + _XMM_SAVE + 16*0], xmm6
        movdqa  [rsp + _XMM_SAVE + 16*1], xmm7
        movdqa  [rsp + _XMM_SAVE + 16*2], xmm8
        movdqa  [rsp + _XMM_SAVE + 16*3], xmm9
        movdqa  [rsp + _XMM_SAVE + 16*4], xmm10
        movdqa  [rsp + _XMM_SAVE + 16*5], xmm11
        movdqa  [rsp + _XMM_SAVE + 16*6], xmm12
        movdqa  [rsp + _XMM_SAVE + 16*7], xmm13
        movdqa  [rsp + _XMM_SAVE + 16*8], xmm14
        movdqa  [rsp + _XMM_SAVE + 16*9], xmm15
%endif

	; if bit (32+3) is set, then all lanes are empty
	mov	unused_lanes, [state + _unused_lanes]
	bt	unused_lanes, 32+3
	jc	return_null

	; find a lane with a non-null job
	xor	idx, idx
	cmp	qword [state + _ldata + 1 * _LANE_DATA_size + _job_in_lane], 0
	cmovne	idx, [one]
	cmp	qword [state + _ldata + 2 * _LANE_DATA_size + _job_in_lane], 0
	cmovne	idx, [two]
	cmp	qword [state + _ldata + 3 * _LANE_DATA_size + _job_in_lane], 0
	cmovne	idx, [three]
	cmp	qword [state + _ldata + 4 * _LANE_DATA_size + _job_in_lane], 0
	cmovne	idx, [four]
	cmp	qword [state + _ldata + 5 * _LANE_DATA_size + _job_in_lane], 0
	cmovne	idx, [five]
	cmp	qword [state + _ldata + 6 * _LANE_DATA_size + _job_in_lane], 0
	cmovne	idx, [six]
	cmp	qword [state + _ldata + 7 * _LANE_DATA_size + _job_in_lane], 0
	cmovne	idx, [seven]

	; copy idx to empty lanes
copy_lane_data:
	mov	tmp, [state + _args + _data_ptr + 8*idx]

%assign I 0
%rep 8
	cmp	qword [state + _ldata + I * _LANE_DATA_size + _job_in_lane], 0
	jne	APPEND(skip_,I)
	mov	[state + _args + _data_ptr + 8*I], tmp
	mov 	dword [state + _lens + 4*I], 0xFFFFFFFF
APPEND(skip_,I):
%assign I (I+1)
%endrep

	; Find min length
	movdqa	xmm0, [state + _lens + 0*16]
        movdqa	xmm1, [state + _lens + 1*16]

	movdqa	xmm2, xmm0
        pminud 	xmm2, xmm1        ; xmm2 has {D,C,B,A}
        palignr xmm3, xmm2, 8     ; xmm3 has {x,x,D,C}
        pminud	xmm2, xmm3        ; xmm2 has {x,x,E,F}
        palignr xmm3, xmm2, 4     ; xmm3 has {x,x,x,E}
        pminud	xmm2, xmm3        ; xmm2 has min value in low dword

	movd   	DWORD(idx), xmm2
 	mov	len2, idx
	and	idx, 0xF
	shr	len2, 4
	jz	len_is_0

	pand   xmm2, [rel clear_low_nibble]
        pshufd xmm2, xmm2, 0

        psubd  xmm0, xmm2
        psubd  xmm1, xmm2

        movdqa [state + _lens + 0*16], xmm0
        movdqa [state + _lens + 1*16], xmm1


	; "state" and "args" are the same address, arg1
	; len is arg2
	call	md5_mb_x4x2_sse
	; state and idx are intact

len_is_0:
	; process completed job "idx"
	imul	lane_data, idx, _LANE_DATA_size
	lea	lane_data, [state + _ldata + lane_data]

	mov	job_rax, [lane_data + _job_in_lane]
	mov	qword [lane_data + _job_in_lane], 0
	mov	dword [job_rax + _status], STS_COMPLETED
	mov	unused_lanes, [state + _unused_lanes]
	shl	unused_lanes, 4
	or	unused_lanes, idx
	mov	[state + _unused_lanes], unused_lanes

	mov	dword [state + _lens + 4*idx], 0xFFFFFFFF
	sub     dword [state + _num_lanes_inuse], 1

	movd	xmm0, [state + _args_digest + 4*idx + 0*32]
	pinsrd	xmm0, [state + _args_digest + 4*idx + 1*32], 1
	pinsrd	xmm0, [state + _args_digest + 4*idx + 2*32], 2
	pinsrd	xmm0, [state + _args_digest + 4*idx + 3*32], 3

	movdqa	[job_rax + _result_digest + 0*16], xmm0

return:

%ifidn __OUTPUT_FORMAT__, win64
        movdqa  xmm6, [rsp + _XMM_SAVE + 16*0]
        movdqa  xmm7, [rsp + _XMM_SAVE + 16*1]
        movdqa  xmm8, [rsp + _XMM_SAVE + 16*2]
        movdqa  xmm9, [rsp + _XMM_SAVE + 16*3]
        movdqa  xmm10, [rsp + _XMM_SAVE + 16*4]
        movdqa  xmm11, [rsp + _XMM_SAVE + 16*5]
        movdqa  xmm12, [rsp + _XMM_SAVE + 16*6]
        movdqa  xmm13, [rsp + _XMM_SAVE + 16*7]
        movdqa  xmm14, [rsp + _XMM_SAVE + 16*8]
        movdqa  xmm15, [rsp + _XMM_SAVE + 16*9]
        mov     rsi, [rsp + _GPR_SAVE + 8*1]
        mov     rdi, [rsp + _GPR_SAVE + 8*2]
%endif
        mov     rbx, [rsp + _GPR_SAVE + 8*0]
        mov     rbp, [rsp + _GPR_SAVE + 8*3]
        mov     r12, [rsp + _GPR_SAVE + 8*4]
        mov     r13, [rsp + _GPR_SAVE + 8*5]
        mov     r14, [rsp + _GPR_SAVE + 8*6]
        mov     r15, [rsp + _GPR_SAVE + 8*7]
        add     rsp, STACK_SPACE

        ret

return_null:
	xor	job_rax, job_rax
	jmp	return


section .data align=16

align 16
clear_low_nibble:
	dq 0x00000000FFFFFFF0, 0x0000000000000000
one:	dq  1
two:	dq  2
three:	dq  3
four:	dq  4
five:	dq  5
six:	dq  6
seven:	dq  7
Commit	Line	Data
7c673cae FG	1	;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
	2	; Copyright(c) 2011-2016 Intel Corporation All rights reserved.
	3	;
	4	; Redistribution and use in source and binary forms, with or without
1e59de90	5	; modification, are permitted provided that the following conditions
7c673cae FG	6	; are met:
	7	; * Redistributions of source code must retain the above copyright
	8	; notice, this list of conditions and the following disclaimer.
	9	; * Redistributions in binary form must reproduce the above copyright
	10	; notice, this list of conditions and the following disclaimer in
	11	; the documentation and/or other materials provided with the
	12	; distribution.
	13	; * Neither the name of Intel Corporation nor the names of its
	14	; contributors may be used to endorse or promote products derived
	15	; from this software without specific prior written permission.
	16	;
	17	; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	18	; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	19	; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	20	; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	21	; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	22	; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	23	; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	24	; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	25	; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	26	; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	27	; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	28	;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
	29
	30	%include "md5_job.asm"
	31	%include "md5_mb_mgr_datastruct.asm"
	32
	33	%include "reg_sizes.asm"
	34
	35	extern md5_mb_x4x2_sse
1e59de90 TL	36
1e59de90 TL	37	[bits 64]
7c673cae	38	default rel
1e59de90	39	section .text
7c673cae FG	40
	41	%if 1
	42	%ifidn __OUTPUT_FORMAT__, elf64
	43	;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
	44	; UN*X register definitions
	45	%define arg1 rdi ; rcx
	46	%define arg2 rsi ; rdx
	47
	48	;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
	49
	50	%else
	51
	52	;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
	53	; WINDOWS register definitions
	54	%define arg1 rcx
	55	%define arg2 rdx
	56
	57	;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
	58	%endif
	59
	60	; Common register definitions
	61
	62	%define state arg1
	63	%define len2 arg2
	64
	65	; idx must be a register not clobberred by md5_mb_x4x2_sse
	66	%define idx r8
	67
	68	%define unused_lanes r9
	69
	70	%define lane_data r10
1e59de90	71
7c673cae FG	72	%define job_rax rax
7c673cae FG	73	%define tmp rax
1e59de90	74
7c673cae FG	75	%endif ;; if 1
	76
	77	; STACK_SPACE needs to be an odd multiple of 8
	78	_XMM_SAVE_SIZE equ 10*16
	79	_GPR_SAVE_SIZE equ 8*8
	80	_ALIGN_SIZE equ 8
	81
	82	_XMM_SAVE equ 0
	83	_GPR_SAVE equ _XMM_SAVE + _XMM_SAVE_SIZE
	84	STACK_SPACE equ _GPR_SAVE + _GPR_SAVE_SIZE + _ALIGN_SIZE
	85
	86	%define APPEND(a,b) a %+ b
	87
	88	; JOB* md5_mb_mgr_flush_sse(MB_MGR_HMAC_OOO *state)
	89	; arg 1 : rcx : state
1e59de90	90	mk_global md5_mb_mgr_flush_sse, function
7c673cae	91	md5_mb_mgr_flush_sse:
1e59de90	92	endbranch
7c673cae FG	93	sub rsp, STACK_SPACE
	94	mov [rsp + _GPR_SAVE + 8*0], rbx
	95	mov [rsp + _GPR_SAVE + 8*3], rbp
	96	mov [rsp + _GPR_SAVE + 8*4], r12
	97	mov [rsp + _GPR_SAVE + 8*5], r13
	98	mov [rsp + _GPR_SAVE + 8*6], r14
	99	mov [rsp + _GPR_SAVE + 8*7], r15
	100	%ifidn __OUTPUT_FORMAT__, win64
	101	mov [rsp + _GPR_SAVE + 8*1], rsi
	102	mov [rsp + _GPR_SAVE + 8*2], rdi
	103	movdqa [rsp + _XMM_SAVE + 16*0], xmm6
	104	movdqa [rsp + _XMM_SAVE + 16*1], xmm7
	105	movdqa [rsp + _XMM_SAVE + 16*2], xmm8
	106	movdqa [rsp + _XMM_SAVE + 16*3], xmm9
	107	movdqa [rsp + _XMM_SAVE + 16*4], xmm10
	108	movdqa [rsp + _XMM_SAVE + 16*5], xmm11
	109	movdqa [rsp + _XMM_SAVE + 16*6], xmm12
	110	movdqa [rsp + _XMM_SAVE + 16*7], xmm13
	111	movdqa [rsp + _XMM_SAVE + 16*8], xmm14
	112	movdqa [rsp + _XMM_SAVE + 16*9], xmm15
	113	%endif
	114
	115	; if bit (32+3) is set, then all lanes are empty
	116	mov unused_lanes, [state + _unused_lanes]
	117	bt unused_lanes, 32+3
	118	jc return_null
	119
	120	; find a lane with a non-null job
	121	xor idx, idx
	122	cmp qword [state + _ldata + 1 * _LANE_DATA_size + _job_in_lane], 0
	123	cmovne idx, [one]
	124	cmp qword [state + _ldata + 2 * _LANE_DATA_size + _job_in_lane], 0
	125	cmovne idx, [two]
	126	cmp qword [state + _ldata + 3 * _LANE_DATA_size + _job_in_lane], 0
	127	cmovne idx, [three]
	128	cmp qword [state + _ldata + 4 * _LANE_DATA_size + _job_in_lane], 0
	129	cmovne idx, [four]
	130	cmp qword [state + _ldata + 5 * _LANE_DATA_size + _job_in_lane], 0
	131	cmovne idx, [five]
	132	cmp qword [state + _ldata + 6 * _LANE_DATA_size + _job_in_lane], 0
	133	cmovne idx, [six]
	134	cmp qword [state + _ldata + 7 * _LANE_DATA_size + _job_in_lane], 0
	135	cmovne idx, [seven]
	136
	137	; copy idx to empty lanes
	138	copy_lane_data:
	139	mov tmp, [state + _args + _data_ptr + 8*idx]
	140
	141	%assign I 0
	142	%rep 8
	143	cmp qword [state + _ldata + I * _LANE_DATA_size + _job_in_lane], 0
	144	jne APPEND(skip_,I)
	145	mov [state + _args + _data_ptr + 8*I], tmp
	146	mov dword [state + _lens + 4*I], 0xFFFFFFFF
	147	APPEND(skip_,I):
	148	%assign I (I+1)
	149	%endrep
	150
	151	; Find min length
	152	movdqa xmm0, [state + _lens + 0*16]
	153	movdqa xmm1, [state + _lens + 1*16]
1e59de90	154
7c673cae FG	155	movdqa xmm2, xmm0
	156	pminud xmm2, xmm1 ; xmm2 has {D,C,B,A}
	157	palignr xmm3, xmm2, 8 ; xmm3 has {x,x,D,C}
	158	pminud xmm2, xmm3 ; xmm2 has {x,x,E,F}
	159	palignr xmm3, xmm2, 4 ; xmm3 has {x,x,x,E}
	160	pminud xmm2, xmm3 ; xmm2 has min value in low dword
1e59de90	161
7c673cae FG	162	movd DWORD(idx), xmm2
	163	mov len2, idx
	164	and idx, 0xF
	165	shr len2, 4
	166	jz len_is_0
1e59de90	167
7c673cae FG	168	pand xmm2, [rel clear_low_nibble]
	169	pshufd xmm2, xmm2, 0
	170
	171	psubd xmm0, xmm2
	172	psubd xmm1, xmm2
	173
	174	movdqa [state + _lens + 0*16], xmm0
	175	movdqa [state + _lens + 1*16], xmm1
	176
	177
	178	; "state" and "args" are the same address, arg1
	179	; len is arg2
	180	call md5_mb_x4x2_sse
	181	; state and idx are intact
	182
	183	len_is_0:
	184	; process completed job "idx"
	185	imul lane_data, idx, _LANE_DATA_size
	186	lea lane_data, [state + _ldata + lane_data]
	187
	188	mov job_rax, [lane_data + _job_in_lane]
	189	mov qword [lane_data + _job_in_lane], 0
	190	mov dword [job_rax + _status], STS_COMPLETED
	191	mov unused_lanes, [state + _unused_lanes]
	192	shl unused_lanes, 4
	193	or unused_lanes, idx
	194	mov [state + _unused_lanes], unused_lanes
	195
	196	mov dword [state + _lens + 4*idx], 0xFFFFFFFF
1e59de90	197	sub dword [state + _num_lanes_inuse], 1
7c673cae FG	198
	199	movd xmm0, [state + _args_digest + 4idx + 032]
	200	pinsrd xmm0, [state + _args_digest + 4idx + 132], 1
	201	pinsrd xmm0, [state + _args_digest + 4idx + 232], 2
	202	pinsrd xmm0, [state + _args_digest + 4idx + 332], 3
	203
	204	movdqa [job_rax + _result_digest + 0*16], xmm0
	205
	206	return:
	207
	208	%ifidn __OUTPUT_FORMAT__, win64
	209	movdqa xmm6, [rsp + _XMM_SAVE + 16*0]
	210	movdqa xmm7, [rsp + _XMM_SAVE + 16*1]
	211	movdqa xmm8, [rsp + _XMM_SAVE + 16*2]
	212	movdqa xmm9, [rsp + _XMM_SAVE + 16*3]
	213	movdqa xmm10, [rsp + _XMM_SAVE + 16*4]
	214	movdqa xmm11, [rsp + _XMM_SAVE + 16*5]
	215	movdqa xmm12, [rsp + _XMM_SAVE + 16*6]
	216	movdqa xmm13, [rsp + _XMM_SAVE + 16*7]
	217	movdqa xmm14, [rsp + _XMM_SAVE + 16*8]
	218	movdqa xmm15, [rsp + _XMM_SAVE + 16*9]
1e59de90 TL	219	mov rsi, [rsp + _GPR_SAVE + 8*1]
1e59de90 TL	220	mov rdi, [rsp + _GPR_SAVE + 8*2]
7c673cae	221	%endif
1e59de90 TL	222	mov rbx, [rsp + _GPR_SAVE + 8*0]
	223	mov rbp, [rsp + _GPR_SAVE + 8*3]
	224	mov r12, [rsp + _GPR_SAVE + 8*4]
	225	mov r13, [rsp + _GPR_SAVE + 8*5]
	226	mov r14, [rsp + _GPR_SAVE + 8*6]
	227	mov r15, [rsp + _GPR_SAVE + 8*7]
7c673cae FG	228	add rsp, STACK_SPACE
	229
	230	ret
	231
	232	return_null:
	233	xor job_rax, job_rax
	234	jmp return
1e59de90	235
7c673cae FG	236
	237	section .data align=16
	238
	239	align 16
	240	clear_low_nibble:
	241	dq 0x00000000FFFFFFF0, 0x0000000000000000
	242	one: dq 1
	243	two: dq 2
	244	three: dq 3
	245	four: dq 4
	246	five: dq 5
	247	six: dq 6
	248	seven: dq 7
	249