[ceph.git] / ceph / src / crypto / isa-l / isa-l_crypto / aes / aarch64 / aes_gcm_aes_init.S

/**********************************************************************
  Copyright(c) 2021 Arm Corporation All rights reserved.

  Redistribution and use in source and binary forms, with or without
  modification, are permitted provided that the following conditions
  are met:
    * Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in
      the documentation and/or other materials provided with the
      distribution.
    * Neither the name of Arm Corporation nor the names of its
      contributors may be used to endorse or promote products derived
      from this software without specific prior written permission.

  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
**********************************************************************/

#include "gcm_common.S"
/*
void gist_aes_gcm_init_##mode(
    const struct gcm_key_data *key_data,
    struct gcm_context_data *context,
    uint8_t *iv,
    uint8_t const *aad,
    uint64_t aad_len
    );
*/
    key_data        .req    x0
    context         .req    x1
    iv              .req    x2
    aad             .req    x3
    aad_len         .req    x4
    temp0           .req    x7
    wtemp0          .req    w7
    temp1           .req    x6
    left_len        .req    x5
    aad_left        .req    x2
    small_tbl_adr   .req    x6

    hashkey_base    .req    x0
    hashkey_addr    .req    x2

    declare_var_vector_reg  AadHash,0
    declare_var_vector_reg  Dat0,1
    declare_var_vector_reg  HashKey0,2
    declare_var_vector_reg  HashKey0Ext,3
    declare_var_vector_reg  High,4
    declare_var_vector_reg  Middle0,5
    declare_var_vector_reg  Low,6
    declare_var_vector_reg  LeftDat,7
    declare_var_vector_reg  Zero,16
    declare_var_vector_reg  Poly,17

    declare_var_vector_reg  Tmp0,18
    declare_var_vector_reg  Tmp1,19
    declare_var_vector_reg  Ctr,1


START_FUNC(init,128,_)
START_FUNC(init,192,_)
START_FUNC(init,256,_)
    stp             aad_len,xzr,[context,AAD_LEN_OFF]           //save in_length and aad_length
    str             xzr,[context,PARTIAL_BLOCK_LENGTH_OFF]      //clear partial_block_length
    add             hashkey_base,key_data,HASHKEY_BASE_OFF
    /* Init Consts for ghash  */
    movi            vZero.4s,0
    mov             temp0,0x87
    dup             vPoly.2d,temp0
    /* Set orig_IV */
    ldr             wtemp0,[iv,8]
    ldr             temp1,[iv]
    movk            temp0,0x100,lsl 48
    stp             temp1,temp0,[context,ORIG_IV_OFF]
    and             left_len,aad_len,15
    ldp             qHashKey0,qHashKey0Ext,[key_data,(HASHKEY_TOTAL_NUM-1)*32]
    /* Set current_counter, save as cpu order */
    ldr             qCtr,[context,ORIG_IV_OFF]
    rev32           vCtr.16b,vCtr.16b
    str             qCtr,[context,CTR_OFF]
    cbz             aad_len,init_zero_exit
    lsr             aad_len,aad_len,4
    /* Read small data */
    cbz             left_len,2f
    add             aad_left,aad,aad_len,lsl 4
    read_small_data_start   LeftDat,aad_left,left_len,small_tbl_adr,Tmp0
    cbz             aad_len,24f     //  aad_len less than 16
2:
    cbnz            left_len,1f
    /*left_len == 0 && aad_len !=0 */

    sub             aad_len,aad_len,1
    /*  leftDat = aad[-1] */
    ldr             qLeftDat,[aad,aad_len,lsl 4]
    cbz             aad_len,24f         /* aad_len == 16 */
1:
    /* aad_len > 16 */
    ldr             qAadHash,[aad],16
    rbit            vAadHash.16b,vAadHash.16b
    sub             aad_len,aad_len,1
1:
    /* loop ghash_block */
    cmp             aad_len,HASHKEY_TOTAL_NUM - 1
    bls             1f /* break loop */
    sub             aad_len,aad_len,HASHKEY_TOTAL_NUM
    ghash_block_n   HASHKEY_TOTAL_NUM,AadHash,Dat0,aad,hashkey_addr,hashkey_base,    \
        HashKey0,HashKey0Ext,High,Low,Middle0,Zero,Poly ,      \
        Tmp0,Tmp1
    b               1b /* back to loop start */
1:
    cbz             aad_len,23f     /* left aad_len == 0 */
    mov             temp0,HASHKEY_TOTAL_NUM - 1
    sub             temp0,temp0,aad_len
    add             hashkey_addr,hashkey_base,temp0,lsl 5
    sub             aad_len,aad_len,1


    ghash_mult_init_round   AadHash,aad,hashkey_addr,HashKey0,HashKey0Ext,  \
        High,Low,Middle0,Tmp0,Dat0,2        /* load next hash */
1:
    cbz             aad_len,1f
    ghash_mult_round        AadHash,aad,hashkey_addr,HashKey0,HashKey0Ext, \
        High,Low,Middle0,Tmp0,Tmp1,Dat0, 2

    sub             aad_len,aad_len,1
    b               1b
1:
    ghash_mult_round_noload AadHash,HashKey0,HashKey0Ext,High,Low,Middle0,Tmp0,Tmp1
    rbit                    vAadHash.16b, vLeftDat.16b
    ghash_mult_final_round  AadHash,High,Low,Middle0,Tmp0,Zero,Poly
    str                     qAadHash,[context]
    ret

23:
    ghash_block_reg AadHash,LeftDat,               \
        HashKey0,HashKey0Ext,High,Low,Middle0,Zero,Poly ,       \
        Tmp0
    str             qAadHash,[context]
    ret
24: /* less or equal than 16 */
    rbit            vLeftDat.16b, vLeftDat.16b
    str             qLeftDat,[context]
    ret
init_zero_exit:
    stp             xzr,xzr,[context]
    ret
END_FUNC(init,128,_)
END_FUNC(init,192,_)
END_FUNC(init,256,_)
Commit	Line	Data
1e59de90 TL	1	/**********************************************************************
	2	Copyright(c) 2021 Arm Corporation All rights reserved.
	3
	4	Redistribution and use in source and binary forms, with or without
	5	modification, are permitted provided that the following conditions
	6	are met:
	7	* Redistributions of source code must retain the above copyright
	8	notice, this list of conditions and the following disclaimer.
	9	* Redistributions in binary form must reproduce the above copyright
	10	notice, this list of conditions and the following disclaimer in
	11	the documentation and/or other materials provided with the
	12	distribution.
	13	* Neither the name of Arm Corporation nor the names of its
	14	contributors may be used to endorse or promote products derived
	15	from this software without specific prior written permission.
	16
	17	THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
	18	"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
	19	LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
	20	A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
	21	OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
	22	SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
	23	LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
	24	DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
	25	THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
	26	(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
	27	OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
	28	**********************************************************************/
	29
	30	#include "gcm_common.S"
	31	/*
	32	void gist_aes_gcm_init_##mode(
	33	const struct gcm_key_data *key_data,
	34	struct gcm_context_data *context,
	35	uint8_t *iv,
	36	uint8_t const *aad,
	37	uint64_t aad_len
	38	);
	39	*/
	40	key_data .req x0
	41	context .req x1
	42	iv .req x2
	43	aad .req x3
	44	aad_len .req x4
	45	temp0 .req x7
	46	wtemp0 .req w7
	47	temp1 .req x6
	48	left_len .req x5
	49	aad_left .req x2
	50	small_tbl_adr .req x6
	51
	52	hashkey_base .req x0
	53	hashkey_addr .req x2
	54
	55	declare_var_vector_reg AadHash,0
	56	declare_var_vector_reg Dat0,1
	57	declare_var_vector_reg HashKey0,2
	58	declare_var_vector_reg HashKey0Ext,3
	59	declare_var_vector_reg High,4
	60	declare_var_vector_reg Middle0,5
	61	declare_var_vector_reg Low,6
	62	declare_var_vector_reg LeftDat,7
	63	declare_var_vector_reg Zero,16
	64	declare_var_vector_reg Poly,17
65
66	declare_var_vector_reg Tmp0,18
67	declare_var_vector_reg Tmp1,19
68	declare_var_vector_reg Ctr,1
69
70
71	START_FUNC(init,128,_)
72	START_FUNC(init,192,_)
73	START_FUNC(init,256,_)
74	stp aad_len,xzr,[context,AAD_LEN_OFF] //save in_length and aad_length
75	str xzr,[context,PARTIAL_BLOCK_LENGTH_OFF] //clear partial_block_length
76	add hashkey_base,key_data,HASHKEY_BASE_OFF
77	/* Init Consts for ghash */
78	movi vZero.4s,0
79	mov temp0,0x87
80	dup vPoly.2d,temp0
81	/* Set orig_IV */
82	ldr wtemp0,[iv,8]
83	ldr temp1,[iv]
84	movk temp0,0x100,lsl 48
85	stp temp1,temp0,[context,ORIG_IV_OFF]
86	and left_len,aad_len,15
87	ldp qHashKey0,qHashKey0Ext,[key_data,(HASHKEY_TOTAL_NUM-1)*32]
88	/* Set current_counter, save as cpu order */
89	ldr qCtr,[context,ORIG_IV_OFF]
90	rev32 vCtr.16b,vCtr.16b
91	str qCtr,[context,CTR_OFF]
92	cbz aad_len,init_zero_exit
93	lsr aad_len,aad_len,4
94	/* Read small data */
95	cbz left_len,2f
96	add aad_left,aad,aad_len,lsl 4
97	read_small_data_start LeftDat,aad_left,left_len,small_tbl_adr,Tmp0
98	cbz aad_len,24f // aad_len less than 16
99	2:
100	cbnz left_len,1f
101	/left_len == 0 && aad_len !=0 /
102
103	sub aad_len,aad_len,1
104	/* leftDat = aad[-1] */
105	ldr qLeftDat,[aad,aad_len,lsl 4]
106	cbz aad_len,24f /* aad_len == 16 */
107	1:
108	/* aad_len > 16 */
109	ldr qAadHash,[aad],16
110	rbit vAadHash.16b,vAadHash.16b
111	sub aad_len,aad_len,1
112	1:
113	/* loop ghash_block */
114	cmp aad_len,HASHKEY_TOTAL_NUM - 1
115	bls 1f /* break loop */
116	sub aad_len,aad_len,HASHKEY_TOTAL_NUM
117	ghash_block_n HASHKEY_TOTAL_NUM,AadHash,Dat0,aad,hashkey_addr,hashkey_base, \
118	HashKey0,HashKey0Ext,High,Low,Middle0,Zero,Poly , \
119	Tmp0,Tmp1
120	b 1b /* back to loop start */
121	1:
122	cbz aad_len,23f /* left aad_len == 0 */
123	mov temp0,HASHKEY_TOTAL_NUM - 1
124	sub temp0,temp0,aad_len
125	add hashkey_addr,hashkey_base,temp0,lsl 5
126	sub aad_len,aad_len,1
127
128
129	ghash_mult_init_round AadHash,aad,hashkey_addr,HashKey0,HashKey0Ext, \
130	High,Low,Middle0,Tmp0,Dat0,2 /* load next hash */
131	1:
132	cbz aad_len,1f
133	ghash_mult_round AadHash,aad,hashkey_addr,HashKey0,HashKey0Ext, \
134	High,Low,Middle0,Tmp0,Tmp1,Dat0, 2
135
136	sub aad_len,aad_len,1
137	b 1b
138	1:
139	ghash_mult_round_noload AadHash,HashKey0,HashKey0Ext,High,Low,Middle0,Tmp0,Tmp1
140	rbit vAadHash.16b, vLeftDat.16b
141	ghash_mult_final_round AadHash,High,Low,Middle0,Tmp0,Zero,Poly
142	str qAadHash,[context]
143	ret
144
145	23:
146	ghash_block_reg AadHash,LeftDat, \
147	HashKey0,HashKey0Ext,High,Low,Middle0,Zero,Poly , \
148	Tmp0
149	str qAadHash,[context]
150	ret
151	24: /* less or equal than 16 */
152	rbit vLeftDat.16b, vLeftDat.16b
153	str qLeftDat,[context]
154	ret
155	init_zero_exit:
156	stp xzr,xzr,[context]
157	ret
158	END_FUNC(init,128,_)
159	END_FUNC(init,192,_)
160	END_FUNC(init,256,_)
161