[ceph.git] / ceph / src / isa-l / crc / aarch64 / crc64_refl_common_pmull.h

########################################################################
#  Copyright(c) 2019 Arm Corporation All rights reserved.
#
#  Redistribution and use in source and binary forms, with or without
#  modification, are permitted provided that the following conditions
#  are met:
#    * Redistributions of source code must retain the above copyright
#      notice, this list of conditions and the following disclaimer.
#    * Redistributions in binary form must reproduce the above copyright
#      notice, this list of conditions and the following disclaimer in
#      the documentation and/or other materials provided with the
#      distribution.
#    * Neither the name of Arm Corporation nor the names of its
#      contributors may be used to endorse or promote products derived
#      from this software without specific prior written permission.
#
#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
#  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
#  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
#  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
#  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
#  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
#  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
#  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#########################################################################

#include "crc_common_pmull.h"

.macro crc64_refl_func name:req
	.arch armv8-a+crypto
	.text
	.align	3
	.global	\name
	.type	\name, %function

/* uint64_t crc64_refl_func(uint64_t seed, const uint8_t * buf, uint64_t len) */

\name\():
	mvn	x_seed, x_seed
	mov	x_counter, 0
	cmp	x_len, (FOLD_SIZE-1)
	bhi	.crc_clmul_pre

.crc_tab_pre:
	cmp	x_len, x_counter
	bls	.done

	adrp	x_tmp, .lanchor_crc_tab
	add	x_buf_iter, x_buf, x_counter
	add	x_buf, x_buf, x_len
	add	x_crc_tab_addr, x_tmp, :lo12:.lanchor_crc_tab

	.align 3
.loop_crc_tab:
	ldrb	w_tmp, [x_buf_iter], 1
	eor	w_tmp, w_tmp, w0
	cmp	x_buf, x_buf_iter
	and	x_tmp, x_tmp, 255
	ldr	x_tmp, [x_crc_tab_addr, x_tmp, lsl 3]
	eor	x_seed, x_tmp, x_seed, lsr 8
	bne	.loop_crc_tab

.done:
	mvn	x_crc_ret, x_seed
	ret

	.align 2
.crc_clmul_pre:
	fmov	d_x0, x_seed // save crc to d_x0

	crc_refl_load_first_block

	bls	.clmul_loop_end

	crc64_load_p4

// 1024bit --> 512bit loop
// merge x0, x1, x2, x3, y0, y1, y2, y3 => x0, x1, x2, x3 (uint64x2_t)
	crc_refl_loop

.clmul_loop_end:
// folding 512bit --> 128bit
	crc64_fold_512b_to_128b

// folding 128bit --> 64bit
	mov	x_tmp, p0_low_b0
	movk	x_tmp, p0_low_b1, lsl 16
	movk	x_tmp, p0_low_b2, lsl 32
	movk	x_tmp, p0_low_b3, lsl 48
	fmov	d_p0_low, x_tmp

	pmull	v_tmp_low.1q, v_x3.1d, v_p0.1d

	mov	d_tmp_high, v_x3.d[1]

	eor	v_x3.16b, v_tmp_high.16b, v_tmp_low.16b

// barrett reduction
	mov	x_tmp, br_low_b0
	movk	x_tmp, br_low_b1, lsl 16
	movk	x_tmp, br_low_b2, lsl 32
	movk	x_tmp, br_low_b3, lsl 48
	fmov	d_br_low, x_tmp

	mov	x_tmp2, br_high_b0
	movk	x_tmp2, br_high_b1, lsl 16
	movk	x_tmp2, br_high_b2, lsl 32
	movk	x_tmp2, br_high_b3, lsl 48
	fmov	d_br_high, x_tmp2

	pmull	v_tmp_low.1q, v_x3.1d, v_br_low.1d
	pmull	v_tmp_high.1q, v_tmp_low.1d, v_br_high.1d

	ext	v_tmp_low.16b, v_br_low.16b, v_tmp_low.16b, #8

	eor	v_tmp_low.16b, v_tmp_low.16b, v_tmp_high.16b
	eor	v_tmp_low.16b, v_tmp_low.16b, v_x3.16b
	umov	x_crc_ret, v_tmp_low.d[1]

	b	.crc_tab_pre

	.size	\name, .-\name
.endm
Commit	Line	Data
f91f0fd5 TL	1	########################################################################
	2	# Copyright(c) 2019 Arm Corporation All rights reserved.
	3	#
	4	# Redistribution and use in source and binary forms, with or without
	5	# modification, are permitted provided that the following conditions
	6	# are met:
	7	# * Redistributions of source code must retain the above copyright
	8	# notice, this list of conditions and the following disclaimer.
	9	# * Redistributions in binary form must reproduce the above copyright
	10	# notice, this list of conditions and the following disclaimer in
	11	# the documentation and/or other materials provided with the
	12	# distribution.
	13	# * Neither the name of Arm Corporation nor the names of its
	14	# contributors may be used to endorse or promote products derived
	15	# from this software without specific prior written permission.
	16	#
	17	# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	18	# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	19	# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	20	# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	21	# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	22	# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	23	# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	24	# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	25	# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	26	# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	27	# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	28	#########################################################################
	29
	30	#include "crc_common_pmull.h"
	31
	32	.macro crc64_refl_func name:req
	33	.arch armv8-a+crypto
	34	.text
	35	.align 3
	36	.global \name
	37	.type \name, %function
	38
	39	/* uint64_t crc64_refl_func(uint64_t seed, const uint8_t * buf, uint64_t len) */
	40
	41	\name\():
	42	mvn x_seed, x_seed
	43	mov x_counter, 0
	44	cmp x_len, (FOLD_SIZE-1)
	45	bhi .crc_clmul_pre
	46
	47	.crc_tab_pre:
	48	cmp x_len, x_counter
	49	bls .done
	50
	51	adrp x_tmp, .lanchor_crc_tab
	52	add x_buf_iter, x_buf, x_counter
	53	add x_buf, x_buf, x_len
	54	add x_crc_tab_addr, x_tmp, :lo12:.lanchor_crc_tab
	55
	56	.align 3
	57	.loop_crc_tab:
	58	ldrb w_tmp, [x_buf_iter], 1
	59	eor w_tmp, w_tmp, w0
	60	cmp x_buf, x_buf_iter
	61	and x_tmp, x_tmp, 255
	62	ldr x_tmp, [x_crc_tab_addr, x_tmp, lsl 3]
	63	eor x_seed, x_tmp, x_seed, lsr 8
	64	bne .loop_crc_tab
65
66	.done:
67	mvn x_crc_ret, x_seed
68	ret
69
70	.align 2
71	.crc_clmul_pre:
72	fmov d_x0, x_seed // save crc to d_x0
73
74	crc_refl_load_first_block
75
76	bls .clmul_loop_end
77
78	crc64_load_p4
79
80	// 1024bit --> 512bit loop
81	// merge x0, x1, x2, x3, y0, y1, y2, y3 => x0, x1, x2, x3 (uint64x2_t)
82	crc_refl_loop
83
84	.clmul_loop_end:
85	// folding 512bit --> 128bit
86	crc64_fold_512b_to_128b
87
88	// folding 128bit --> 64bit
89	mov x_tmp, p0_low_b0
90	movk x_tmp, p0_low_b1, lsl 16
91	movk x_tmp, p0_low_b2, lsl 32
92	movk x_tmp, p0_low_b3, lsl 48
93	fmov d_p0_low, x_tmp
94
95	pmull v_tmp_low.1q, v_x3.1d, v_p0.1d
96
97	mov d_tmp_high, v_x3.d[1]
98
99	eor v_x3.16b, v_tmp_high.16b, v_tmp_low.16b
100
101	// barrett reduction
102	mov x_tmp, br_low_b0
103	movk x_tmp, br_low_b1, lsl 16
104	movk x_tmp, br_low_b2, lsl 32
105	movk x_tmp, br_low_b3, lsl 48
106	fmov d_br_low, x_tmp
107
108	mov x_tmp2, br_high_b0
109	movk x_tmp2, br_high_b1, lsl 16
110	movk x_tmp2, br_high_b2, lsl 32
111	movk x_tmp2, br_high_b3, lsl 48
112	fmov d_br_high, x_tmp2
113
114	pmull v_tmp_low.1q, v_x3.1d, v_br_low.1d
115	pmull v_tmp_high.1q, v_tmp_low.1d, v_br_high.1d
116
117	ext v_tmp_low.16b, v_br_low.16b, v_tmp_low.16b, #8
118
119	eor v_tmp_low.16b, v_tmp_low.16b, v_tmp_high.16b
120	eor v_tmp_low.16b, v_tmp_low.16b, v_x3.16b
121	umov x_crc_ret, v_tmp_low.d[1]
122
123	b .crc_tab_pre
124
125	.size \name, .-\name
126	.endm