projects / ceph.git / blame
| Commit | Line | Data |
|---|---|---|
| 1e59de90 TL |
1 | /********************************************************************** |
| 2 | Copyright(c) 2021 Arm Corporation All rights reserved. | |
| 3 | ||
| 4 | Redistribution and use in source and binary forms, with or without | |
| 5 | modification, are permitted provided that the following conditions | |
| 6 | are met: | |
| 7 | * Redistributions of source code must retain the above copyright | |
| 8 | notice, this list of conditions and the following disclaimer. | |
| 9 | * Redistributions in binary form must reproduce the above copyright | |
| 10 | notice, this list of conditions and the following disclaimer in | |
| 11 | the documentation and/or other materials provided with the | |
| 12 | distribution. | |
| 13 | * Neither the name of Arm Corporation nor the names of its | |
| 14 | contributors may be used to endorse or promote products derived | |
| 15 | from this software without specific prior written permission. | |
| 16 | ||
| 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
| 18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
| 19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
| 20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
| 21 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
| 22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
| 23 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
| 24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
| 25 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
| 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
| 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
| 28 | **********************************************************************/ | |
| 29 | /* | |
| 30 | void gist_aes_gcm_dec_##mode( \ | |
| 31 | const struct gcm_key_data *key_data, \ | |
| 32 | struct gcm_context_data *context, \ | |
| 33 | uint8_t *out, \ | |
| 34 | uint8_t const *in, \ | |
| 35 | uint64_t len, \ | |
| 36 | uint8_t *iv, \ | |
| 37 | \ | |
| 38 | uint8_t const *aad, \ | |
| 39 | uint64_t aad_len, \ | |
| 40 | uint8_t *auth_tag, \ | |
| 41 | uint64_t auth_tag_len \ | |
| 42 | \ | |
| 43 | ) | |
| 44 | */ | |
| 45 | ||
| 46 | declare_var_generic_reg key_data ,0 | |
| 47 | declare_var_generic_reg context ,1 | |
| 48 | declare_var_generic_reg out ,2 | |
| 49 | declare_var_generic_reg in ,3 | |
| 50 | declare_var_generic_reg len ,4 | |
| 51 | declare_var_generic_reg iv ,5 | |
| 52 | declare_var_generic_reg aad ,6 | |
| 53 | declare_var_generic_reg aad_len ,7 | |
| 54 | ||
| 55 | declare_var_generic_reg hashkey_base,0 | |
| 56 | declare_var_generic_reg hashkey_addr,5 | |
| 57 | declare_var_generic_reg left_len ,12 | |
| 58 | declare_var_generic_reg aad_left ,13 | |
| 59 | declare_var_generic_reg temp0 ,14 | |
| 60 | declare_var_generic_reg temp1 ,15 | |
| 61 | ||
| 62 | declare_var_generic_reg auth_tag ,0 /* input param */ | |
| 63 | declare_var_generic_reg auth_tag_len,1 /* input param */ | |
| 64 | ||
| 65 | ||
| 66 | declare_var_vector_reg Ctr,0 | |
| 67 | declare_var_vector_reg AadHash,1 | |
| 68 | declare_var_vector_reg HashKey0,2 | |
| 69 | declare_var_vector_reg HashKey0Ext,3 | |
| 70 | declare_var_vector_reg High,4 | |
| 71 | declare_var_vector_reg Low,5 | |
| 72 | declare_var_vector_reg EncCtr,6 | |
| 73 | declare_var_vector_reg Dat0,6 | |
| 74 | declare_var_vector_reg Middle0,7 | |
| 75 | ||
| 76 | declare_var_vector_reg Tmp0,8 | |
| 77 | declare_var_vector_reg Tmp1,9 | |
| 78 | declare_var_vector_reg Zero,10 | |
| 79 | declare_var_vector_reg Poly,11 | |
| 80 | declare_var_vector_reg LeftDat ,12 | |
| 81 | declare_var_vector_reg Len ,13 | |
| 82 | declare_var_vector_reg Tmp2,14 | |
| 83 | declare_var_vector_reg Tmp3,15 | |
| 84 | ||
| 85 | declare_var_vector_reg One,31 | |
| 86 | .set stack_size,64 | |
| 87 | .macro push_stack | |
| 88 | stp d8, d9,[sp,-stack_size]! | |
| 89 | stp d10,d11,[sp,16] | |
| 90 | stp d12,d13,[sp,32] | |
| 91 | stp d14,d15,[sp,48] | |
| 92 | ||
| 93 | .endm | |
| 94 | ||
| 95 | .macro pop_stack | |
| 96 | ldp d10,d11,[sp,16] | |
| 97 | ldp d12,d13,[sp,32] | |
| 98 | ldp d14,d15,[sp,48] | |
| 99 | ldp d8, d9, [sp], stack_size | |
| 100 | .endm | |
| 101 | ||
| 102 | START_FUNC(enc,KEY_LEN,_) | |
| 103 | START_FUNC(enc,KEY_LEN,_nt_) | |
| 104 | push_stack | |
| 105 | /*save in_length and aad_length*/ | |
| 106 | stp aad_len,len,[context,AAD_LEN_OFF] | |
| 107 | load_aes_keys key_data | |
| 108 | /* Init Consts and IV */ | |
| 109 | mov wtemp1,1 | |
| 110 | eor vOne.16b,vOne.16b,vOne.16b | |
| 111 | ld1 {vCtr.d}[0],[iv],8 | |
| 112 | eor vZero.16b,vZero.16b,vZero.16b | |
| 113 | ld1 {vCtr.s}[2],[iv] | |
| 114 | mov temp0,0x87 | |
| 115 | rev32 vCtr.16b,vCtr.16b /* to cpu order */ | |
| 116 | ins vOne.s[3],wtemp1 | |
| 117 | mov vAadHash.16b,vZero.16b | |
| 118 | dup vPoly.2d,temp0 | |
| 119 | ins vCtr.s[3],wtemp1 /* Initial Ctr and Orig IV */ | |
| 120 | ||
| 121 | ||
| 122 | and left_len,aad_len,0xf | |
| 123 | cbz aad_len,24f | |
| 124 | lsr aad_len,aad_len,4 | |
| 125 | /* Read small data */ | |
| 126 | cbz left_len,2f /* aad_len >= 16,skip */ | |
| 127 | add aad_left,aad,aad_len,lsl 4 | |
| 128 | read_small_data_start LeftDat,aad_left,left_len,temp0,Tmp0 | |
| 129 | cbnz left_len,1f /* aad_len & 0xf != 0 */ | |
| 130 | 2: | |
| 131 | cbz aad_len,1f /* aad_len <16 skip*/ | |
| 132 | /* left_len == 0 && aad_len !=0 */ | |
| 133 | sub aad_len,aad_len,1 | |
| 134 | /* leftDat = aad[-1] */ | |
| 135 | ldr qLeftDat,[aad,aad_len,lsl 4] | |
| 136 | 1: | |
| 137 | cbnz aad_len,1f /* aad_len >16,skip */ | |
| 138 | rbit vAadHash.16b,vLeftDat.16b | |
| 139 | b 24f /* aad_len <=16, skip aadhash caculate */ | |
| 140 | 1: | |
| 141 | /* aad_len > 16 */ | |
| 142 | ldr qAadHash,[aad],16 | |
| 143 | rbit vAadHash.16b,vAadHash.16b | |
| 144 | sub aad_len,aad_len,1 | |
| 145 | ||
| 146 | 1: | |
| 147 | /* loop ghash_block */ | |
| 148 | cmp aad_len,HASHKEY_TOTAL_NUM - 1 | |
| 149 | bls 1f // break loop | |
| 150 | sub aad_len,aad_len,HASHKEY_TOTAL_NUM | |
| 151 | ghash_block_n HASHKEY_TOTAL_NUM,AadHash,Dat0,aad,hashkey_addr,hashkey_base, \ | |
| 152 | HashKey0,HashKey0Ext,High,Low,Middle0,Zero,Poly , \ | |
| 153 | Tmp0,Tmp1 | |
| 154 | b 1b /* back to loop start */ | |
| 155 | 1: | |
| 156 | cbnz aad_len,1f /* left aad_len >32,skip */ | |
| 157 | ldp qHashKey0,qHashKey0Ext,[hashkey_base,(HASHKEY_TOTAL_NUM-1)*32] | |
| 158 | ghash_block_reg AadHash,LeftDat, \ | |
| 159 | HashKey0,HashKey0Ext,High,Low,Middle0,Zero,Poly , \ | |
| 160 | Tmp0 | |
| 161 | b 24f /* left aad_len <=32,skip below check */ | |
| 162 | 1: | |
| 163 | mov temp0,HASHKEY_TOTAL_NUM - 1 | |
| 164 | sub temp0,temp0,aad_len | |
| 165 | add hashkey_addr,hashkey_base,temp0,lsl 5 | |
| 166 | ||
| 167 | ghash_mult_init_round AadHash,aad,hashkey_addr,HashKey0,HashKey0Ext, \ | |
| 168 | High,Low,Middle0,Tmp0,Dat0,2 /* load next hash */ | |
| 169 | sub aad_len,aad_len,1 | |
| 170 | ||
| 171 | 1: | |
| 172 | cbz aad_len,1f | |
| 173 | ghash_mult_round AadHash,aad,hashkey_addr,HashKey0,HashKey0Ext, \ | |
| 174 | High,Low,Middle0,Tmp0,Tmp1,Dat0, 2 | |
| 175 | ||
| 176 | sub aad_len,aad_len,1 | |
| 177 | b 1b | |
| 178 | 1: | |
| 179 | ghash_mult_round_noload AadHash,HashKey0,HashKey0Ext,High,Low,Middle0,Tmp0,Tmp1 | |
| 180 | rbit vAadHash.16b, vLeftDat.16b | |
| 181 | ghash_mult_final_round AadHash,High,Low,Middle0,Tmp0,Zero,Poly | |
| 182 | ||
| 183 | 24: | |
| 184 | ||
| 185 | /* Enc/Dec loop */ | |
| 186 | and left_len,len,15 | |
| 187 | cbz len,24f | |
| 188 | lsr len,len,4 | |
| 189 | 1: | |
| 190 | /* loop aes gcm enc/dec loop */ | |
| 191 | cmp len,HASHKEY_TOTAL_NUM - 1 | |
| 192 | bls 1f // break loop | |
| 193 | sub len,len,HASHKEY_TOTAL_NUM | |
| 194 | aes_gcm_n_round encrypt,HASHKEY_TOTAL_NUM,AadHash,in,hashkey_addr,hashkey_base, \ | |
| 195 | HashKey0,HashKey0Ext,High,Low,Poly, \ | |
| 196 | Ctr,EncCtr,One,out,Tmp0,Tmp1 | |
| 197 | b 1b /* back to loop start */ | |
| 198 | 1: | |
| 199 | cbz len,24f /* left len == 0 */ | |
| 200 | mov temp0,HASHKEY_TOTAL_NUM | |
| 201 | sub temp0,temp0,len | |
| 202 | add hashkey_addr,hashkey_base,temp0,lsl 5 | |
| 203 | ||
| 204 | sub len,len,1 | |
| 205 | aes_gcm_init encrypt,AadHash,in,hashkey_addr,HashKey0,HashKey0Ext, \ | |
| 206 | High,Low,Ctr,EncCtr,One,out,Tmp0,Tmp1,2 /* load next hash */ | |
| 207 | cbz len,2f | |
| 208 | sub len,len,1 | |
| 209 | 1: | |
| 210 | ||
| 211 | cbz len,1f | |
| 212 | aes_gcm_middle encrypt,AadHash,in,hashkey_addr,HashKey0,HashKey0Ext, \ | |
| 213 | High,Low,Ctr,EncCtr,One,out,Tmp0,Tmp1,2 /* load next hash */ | |
| 214 | sub len,len,1 | |
| 215 | b 1b | |
| 216 | 1: | |
| 217 | aes_gcm_middle encrypt,AadHash,in,hashkey_addr,HashKey0,HashKey0Ext, \ | |
| 218 | High,Low,Ctr,EncCtr,One,out,Tmp0,Tmp1,1 /* load next hash */ | |
| 219 | 2: | |
| 220 | poly_mult_final_x2 AadHash,High,Low,Tmp0,Tmp1,Poly | |
| 221 | 24: | |
| 222 | /* complete part */ | |
| 223 | cmp left_len,0 | |
| 224 | movi vHigh.16b,0 | |
| 225 | mov temp0,HASHKEY_TOTAL_NUM-3 | |
| 226 | movi vLow.16b,0 | |
| 227 | cinc hashkey_addr,temp0,eq | |
| 228 | movi vMiddle0.16b,0 | |
| 229 | add hashkey_addr,hashkey_base,hashkey_addr,lsl 5 | |
| 230 | ldp qHashKey0,qHashKey0Ext,[hashkey_addr],32 | |
| 231 | beq 2f | |
| 232 | read_small_data_start LeftDat,in,left_len,temp0,Tmp0 | |
| 233 | add vCtr.4s,vCtr.4s,vOne.4s | |
| 234 | rev32 vEncCtr.16b,vCtr.16b | |
| 235 | aes_encrypt_round EncCtr,Key0 | |
| 236 | pmull2 vHigh.1q,vAadHash.2d,vHashKey0.2d | |
| 237 | aes_encrypt_round EncCtr,Key1 | |
| 238 | pmull vLow.1q ,vAadHash.1d,vHashKey0.1d | |
| 239 | aes_encrypt_round EncCtr,Key2 | |
| 240 | ldr qHashKey0,[hashkey_addr],16 | |
| 241 | aes_encrypt_round EncCtr,Key3 | |
| 242 | pmull vMiddle0.1q,vAadHash.1d,vHashKey0Ext.1d | |
| 243 | aes_encrypt_round EncCtr,Key4 | |
| 244 | pmull2 vTmp0.1q ,vAadHash.2d,vHashKey0Ext.2d | |
| 245 | aes_encrypt_round EncCtr,Key5 | |
| 246 | ldr qHashKey0Ext,[hashkey_addr],16 | |
| 247 | aes_encrypt_round EncCtr,Key6 | |
| 248 | eor vMiddle0.16b,vMiddle0.16b,vTmp0.16b | |
| 249 | aes_encrypt_round EncCtr,Key7 | |
| 250 | aes_encrypt_round EncCtr,Key8 | |
| 251 | #if KEY_LEN==256 | |
| 252 | aes_encrypt_round EncCtr,Key9 | |
| 253 | aes_encrypt_round EncCtr,Key10 | |
| 254 | aes_encrypt_round EncCtr,Key11 | |
| 255 | aes_encrypt_round EncCtr,Key12 | |
| 256 | aese vEncCtr.16b,vKey13.16b | |
| 257 | eor vEncCtr.16b,vEncCtr.16b,vKey14.16b | |
| 258 | #else | |
| 259 | aese vEncCtr.16b,vKey9.16b | |
| 260 | eor vEncCtr.16b,vEncCtr.16b,vKey10.16b | |
| 261 | #endif | |
| 262 | eor vEncCtr.16b,vEncCtr.16b,vLeftDat.16b | |
| 263 | write_small_data_start EncCtr,out,left_len,temp0,Tmp0 | |
| 264 | clear_small_data EncCtr,Zero,left_len,temp0,Tmp0 | |
| 265 | rbit vAadHash.16b,vEncCtr.16b | |
| 266 | 2: | |
| 267 | ||
| 268 | ldr qLen,[context,AAD_LEN_OFF] /* Len */ | |
| 269 | mov wtemp0,1 /* Ek */ | |
| 270 | pmull2 vTmp0.1q ,vAadHash.2d,vHashKey0.2d /* auth_dat * HashKey[Total-2] */ | |
| 271 | shl vLen.2d,vLen.2d,3 /* Len */ | |
| 272 | pmull vTmp1.1q ,vAadHash.1d,vHashKey0.1d /* auth_dat * HashKey[Total-2] */ | |
| 273 | rev64 vLen.16b,vLen.16b /* Len */ | |
| 274 | ins vCtr.4s[3],wtemp0 /* Ek */ | |
| 275 | ldr qHashKey0,[hashkey_addr],16 /* auth_dat * HashKey[Total-2] */ | |
| 276 | pmull vTmp2.1q,vAadHash.1d,vHashKey0Ext.1d /* auth_dat * HashKey[Total-2] */ | |
| 277 | rev32 vEncCtr.16b,vCtr.16b /* Ek */ | |
| 278 | eor vHigh.16b,vHigh.16b,vTmp0.16b /* auth_dat * HashKey[Total-2] */ | |
| 279 | pmull2 vTmp3.1q ,vAadHash.2d,vHashKey0Ext.2d /* auth_dat * HashKey[Total-2] */ | |
| 280 | rbit vAadHash.16b,vLen.16b /* Len */ | |
| 281 | ||
| 282 | aes_encrypt_round EncCtr,Key0 /* Ek */ | |
| 283 | eor vLow.16b,vLow.16b,vTmp1.16b /* auth_dat * HashKey[Total-2] */ | |
| 284 | aes_encrypt_round EncCtr,Key1 /* Ek */ | |
| 285 | ldr qHashKey0Ext,[hashkey_addr],16 /* auth_dat * HashKey[Total-2] */ | |
| 286 | aes_encrypt_round EncCtr,Key2 /* Ek */ | |
| 287 | eor vMiddle0.16b,vMiddle0.16b,vTmp2.16b /* auth_dat * HashKey[Total-2] */ | |
| 288 | aes_encrypt_round EncCtr,Key3 /* Ek */ | |
| 289 | eor vMiddle0.16b,vMiddle0.16b,vTmp3.16b /* auth_dat * HashKey[Total-2] */ | |
| 290 | aes_encrypt_round EncCtr,Key4 /* Ek */ | |
| 291 | ||
| 292 | pmull2 vTmp0.1q,vAadHash.2d,vHashKey0.2d /* Len * HashKey[Total-1] */ | |
| 293 | pmull vTmp1.1q ,vAadHash.1d,vHashKey0.1d /* Len * HashKey[Total-1] */ | |
| 294 | aes_encrypt_round EncCtr,Key5 /* Ek */ | |
| 295 | aes_encrypt_round EncCtr,Key6 /* Ek */ | |
| 296 | pmull vTmp2.1q,vAadHash.1d,vHashKey0Ext.1d /* Len * HashKey[Total-1] */ | |
| 297 | aes_encrypt_round EncCtr,Key7 /* Ek */ | |
| 298 | eor vHigh.16b,vHigh.16b,vTmp0.16b /* Len * HashKey[Total-1] */ | |
| 299 | pmull2 vTmp3.1q ,vAadHash.2d,vHashKey0Ext.2d /* Len * HashKey[Total-1] */ | |
| 300 | aes_encrypt_round EncCtr,Key8 /* Ek */ | |
| 301 | eor vLow.16b,vLow.16b,vTmp1.16b /* Len * HashKey[Total-1] */ | |
| 302 | #if KEY_LEN==256 | |
| 303 | aes_encrypt_round EncCtr,Key9 /* Ek */ | |
| 304 | aes_encrypt_round EncCtr,Key10 /* Ek */ | |
| 305 | aes_encrypt_round EncCtr,Key11 /* Ek */ | |
| 306 | aes_encrypt_round EncCtr,Key12 /* Ek */ | |
| 307 | aese vEncCtr.16b,vKey13.16b /* Ek */ | |
| 308 | eor vEncCtr.16b,vEncCtr.16b,vKey14.16b /* Ek */ | |
| 309 | #else | |
| 310 | aese vEncCtr.16b,vKey9.16b /* Ek */ | |
| 311 | eor vEncCtr.16b,vEncCtr.16b,vKey10.16b /* Ek */ | |
| 312 | #endif | |
| 313 | eor vMiddle0.16b,vMiddle0.16b,vTmp2.16b /* Len * HashKey[Total-1] */ | |
| 314 | eor vMiddle0.16b,vMiddle0.16b,vTmp3.16b /* Len * HashKey[Total-1] */ | |
| 315 | rbit vAadHash.16b,vEncCtr.16b /* Aad */ | |
| 316 | ||
| 317 | ghash_mult_final_round AadHash,High,Low,Middle0,Tmp0,Zero,Poly | |
| 318 | ||
| 319 | ldp auth_tag,auth_tag_len,[sp,stack_size] /* Adjust here : TODO TBD */ | |
| 320 | rbit vAadHash.16b,vAadHash.16b /* Aad */ | |
| 321 | ||
| 322 | ||
| 323 | /* output auth_tag */ | |
| 324 | cmp auth_tag_len,16 | |
| 325 | bne 1f | |
| 326 | /* most likely auth_tag_len=16 */ | |
| 327 | str qAadHash,[auth_tag] | |
| 328 | pop_stack | |
| 329 | ret | |
| 330 | 1: /* auth_tag_len=12 */ | |
| 331 | cmp auth_tag_len,12 | |
| 332 | bne 1f | |
| 333 | str dAadHash,[auth_tag],8 | |
| 334 | st1 {vAadHash.s}[2],[auth_tag] | |
| 335 | pop_stack | |
| 336 | ret | |
| 337 | 1: /* auth_tag_len=8 */ | |
| 338 | str dAadHash,[auth_tag] | |
| 339 | pop_stack | |
| 340 | ret | |
| 341 | END_FUNC(enc,KEY_LEN,_) | |
| 342 | END_FUNC(enc,KEY_LEN,_nt_) | |
| 343 | ||
| 344 | ||
| 345 | START_FUNC(dec,KEY_LEN,_) | |
| 346 | START_FUNC(dec,KEY_LEN,_nt_) | |
| 347 | push_stack | |
| 348 | /* save in_length and aad_length */ | |
| 349 | stp aad_len,len,[context,AAD_LEN_OFF] | |
| 350 | load_aes_keys key_data | |
| 351 | /* Init Consts and IV */ | |
| 352 | mov wtemp1,1 | |
| 353 | eor vOne.16b,vOne.16b,vOne.16b | |
| 354 | ld1 {vCtr.d}[0],[iv],8 | |
| 355 | eor vZero.16b,vZero.16b,vZero.16b | |
| 356 | ld1 {vCtr.s}[2],[iv] | |
| 357 | mov temp0,0x87 | |
| 358 | rev32 vCtr.16b,vCtr.16b /* to cpu order */ | |
| 359 | mov vAadHash.16b,vZero.16b | |
| 360 | ins vOne.s[3],wtemp1 | |
| 361 | dup vPoly.2d,temp0 | |
| 362 | ins vCtr.s[3],wtemp1 /* Initial Ctr and Orig IV */ | |
| 363 | ||
| 364 | ldp qHashKey0,qHashKey0Ext,[hashkey_base] | |
| 365 | and left_len,aad_len,0xf | |
| 366 | cbz aad_len,24f | |
| 367 | lsr aad_len,aad_len,4 | |
| 368 | /* Read small data */ | |
| 369 | cbz left_len,2f /* aad_len >= 16,skip */ | |
| 370 | add aad_left,aad,aad_len,lsl 4 | |
| 371 | read_small_data_start LeftDat,aad_left,left_len,temp0,Tmp0 | |
| 372 | cbnz left_len,1f /* aad_len & 0xf != 0 */ | |
| 373 | 2: | |
| 374 | cbz aad_len,1f /* aad_len <16 skip */ | |
| 375 | /* left_len == 0 && aad_len !=0 */ | |
| 376 | sub aad_len,aad_len,1 | |
| 377 | /* leftDat = aad[-1] */ | |
| 378 | ldr qLeftDat,[aad,aad_len,lsl 4] | |
| 379 | 1: | |
| 380 | cbnz aad_len,1f /* aad_len >16,skip */ | |
| 381 | rbit vAadHash.16b,vLeftDat.16b | |
| 382 | b 24f /* aad_len <=16, skip aadhash caculate */ | |
| 383 | 1: | |
| 384 | /* aad_len > 16 */ | |
| 385 | ldr qAadHash,[aad],16 | |
| 386 | rbit vAadHash.16b,vAadHash.16b | |
| 387 | sub aad_len,aad_len,1 | |
| 388 | ||
| 389 | 1: | |
| 390 | /** loop ghash_block */ | |
| 391 | cmp aad_len,HASHKEY_TOTAL_NUM - 1 | |
| 392 | bls 1f /* break loop */ | |
| 393 | sub aad_len,aad_len,HASHKEY_TOTAL_NUM | |
| 394 | ghash_block_n HASHKEY_TOTAL_NUM,AadHash,Dat0,aad,hashkey_addr,hashkey_base, \ | |
| 395 | HashKey0,HashKey0Ext,High,Low,Middle0,Zero,Poly , \ | |
| 396 | Tmp0,Tmp1 | |
| 397 | b 1b /* back to loop start */ | |
| 398 | 1: | |
| 399 | cbnz aad_len,1f /* left aad_len >32,skip */ | |
| 400 | ldp qHashKey0,qHashKey0Ext,[hashkey_base,(HASHKEY_TOTAL_NUM-1)*32] | |
| 401 | ghash_block_reg AadHash,LeftDat, \ | |
| 402 | HashKey0,HashKey0Ext,High,Low,Middle0,Zero,Poly , \ | |
| 403 | Tmp0 | |
| 404 | b 24f /* left aad_len <=32,skip below check */ | |
| 405 | 1: | |
| 406 | mov temp0,HASHKEY_TOTAL_NUM - 1 | |
| 407 | sub temp0,temp0,aad_len | |
| 408 | add hashkey_addr,hashkey_base,temp0,lsl 5 | |
| 409 | ||
| 410 | ghash_mult_init_round AadHash,aad,hashkey_addr,HashKey0,HashKey0Ext, \ | |
| 411 | High,Low,Middle0,Tmp0,Dat0,2 /* load next hash */ | |
| 412 | sub aad_len,aad_len,1 | |
| 413 | ||
| 414 | 1: | |
| 415 | cbz aad_len,1f | |
| 416 | ghash_mult_round AadHash,aad,hashkey_addr,HashKey0,HashKey0Ext, \ | |
| 417 | High,Low,Middle0,Tmp0,Tmp1,Dat0, 2 | |
| 418 | ||
| 419 | sub aad_len,aad_len,1 | |
| 420 | b 1b | |
| 421 | 1: | |
| 422 | ghash_mult_round_noload AadHash,HashKey0,HashKey0Ext,High,Low,Middle0,Tmp0,Tmp1 | |
| 423 | rbit vAadHash.16b, vLeftDat.16b | |
| 424 | ghash_mult_final_round AadHash,High,Low,Middle0,Tmp0,Zero,Poly | |
| 425 | ||
| 426 | 24: | |
| 427 | ||
| 428 | ||
| 429 | /* Enc/Dec loop */ | |
| 430 | and left_len,len,15 | |
| 431 | cbz len,24f | |
| 432 | lsr len,len,4 | |
| 433 | 1: | |
| 434 | /* loop aes gcm enc/dec loop */ | |
| 435 | cmp len,HASHKEY_TOTAL_NUM - 1 | |
| 436 | bls 1f // break loop | |
| 437 | sub len,len,HASHKEY_TOTAL_NUM | |
| 438 | aes_gcm_n_round decrypt,HASHKEY_TOTAL_NUM,AadHash,in,hashkey_addr,hashkey_base, \ | |
| 439 | HashKey0,HashKey0Ext,High,Low,Poly, \ | |
| 440 | Ctr,EncCtr,One,out,Tmp0,Tmp1 | |
| 441 | b 1b /* back to loop start */ | |
| 442 | 1: | |
| 443 | cbz len,24f /* left len == 0 */ | |
| 444 | mov temp0,HASHKEY_TOTAL_NUM | |
| 445 | sub temp0,temp0,len | |
| 446 | add hashkey_addr,hashkey_base,temp0,lsl 5 | |
| 447 | ||
| 448 | sub len,len,1 | |
| 449 | aes_gcm_init decrypt,AadHash,in,hashkey_addr,HashKey0,HashKey0Ext, \ | |
| 450 | High,Low,Ctr,EncCtr,One,out,Tmp0,Tmp1,2 /* load next hash */ | |
| 451 | cbz len,2f | |
| 452 | sub len,len,1 | |
| 453 | 1: | |
| 454 | ||
| 455 | cbz len,1f | |
| 456 | aes_gcm_middle decrypt,AadHash,in,hashkey_addr,HashKey0,HashKey0Ext, \ | |
| 457 | High,Low,Ctr,EncCtr,One,out,Tmp0,Tmp1,2 /* load next hash */ | |
| 458 | sub len,len,1 | |
| 459 | b 1b | |
| 460 | 1: | |
| 461 | aes_gcm_middle decrypt,AadHash,in,hashkey_addr,HashKey0,HashKey0Ext, \ | |
| 462 | High,Low,Ctr,EncCtr,One,out,Tmp0,Tmp1,1 /* load next hash */ | |
| 463 | 2: | |
| 464 | poly_mult_final_x2 AadHash,High,Low,Tmp0,Tmp1,Poly | |
| 465 | 24: | |
| 466 | /* complete part */ | |
| 467 | cmp left_len,0 | |
| 468 | movi vHigh.16b,0 | |
| 469 | mov temp0,21 | |
| 470 | movi vLow.16b,0 | |
| 471 | cinc hashkey_addr,temp0,eq | |
| 472 | movi vMiddle0.16b,0 | |
| 473 | add hashkey_addr,hashkey_base,hashkey_addr,lsl 5 | |
| 474 | ldp qHashKey0,qHashKey0Ext,[hashkey_addr],32 | |
| 475 | beq 2f | |
| 476 | read_small_data_start LeftDat,in,left_len,temp0,Tmp0 | |
| 477 | add vCtr.4s,vCtr.4s,vOne.4s | |
| 478 | rev32 vEncCtr.16b,vCtr.16b | |
| 479 | aes_encrypt_round EncCtr,Key0 | |
| 480 | pmull2 vHigh.1q,vAadHash.2d,vHashKey0.2d | |
| 481 | aes_encrypt_round EncCtr,Key1 | |
| 482 | pmull vLow.1q ,vAadHash.1d,vHashKey0.1d | |
| 483 | aes_encrypt_round EncCtr,Key2 | |
| 484 | ldr qHashKey0,[hashkey_addr],16 | |
| 485 | aes_encrypt_round EncCtr,Key3 | |
| 486 | pmull vMiddle0.1q,vAadHash.1d,vHashKey0Ext.1d | |
| 487 | aes_encrypt_round EncCtr,Key4 | |
| 488 | pmull2 vTmp0.1q ,vAadHash.2d,vHashKey0Ext.2d | |
| 489 | aes_encrypt_round EncCtr,Key5 | |
| 490 | ldr qHashKey0Ext,[hashkey_addr],16 | |
| 491 | aes_encrypt_round EncCtr,Key6 | |
| 492 | eor vMiddle0.16b,vMiddle0.16b,vTmp0.16b | |
| 493 | aes_encrypt_round EncCtr,Key7 | |
| 494 | aes_encrypt_round EncCtr,Key8 | |
| 495 | #if KEY_LEN==256 | |
| 496 | aes_encrypt_round EncCtr,Key9 | |
| 497 | aes_encrypt_round EncCtr,Key10 | |
| 498 | aes_encrypt_round EncCtr,Key11 | |
| 499 | aes_encrypt_round EncCtr,Key12 | |
| 500 | aese vEncCtr.16b,vKey13.16b | |
| 501 | eor vEncCtr.16b,vEncCtr.16b,vKey14.16b | |
| 502 | eor vEncCtr.16b,vEncCtr.16b,vLeftDat.16b | |
| 503 | #endif | |
| 504 | #if KEY_LEN==128 | |
| 505 | aese vEncCtr.16b,vKey9.16b | |
| 506 | eor vEncCtr.16b,vEncCtr.16b,vKey10.16b | |
| 507 | eor vEncCtr.16b,vEncCtr.16b,vLeftDat.16b | |
| 508 | #endif | |
| 509 | write_small_data_start EncCtr,out,left_len,temp0,Tmp0 | |
| 510 | rbit vAadHash.16b,vLeftDat.16b | |
| 511 | ||
| 512 | 2: | |
| 513 | ||
| 514 | ldr qLen,[context,AAD_LEN_OFF] /* Len */ | |
| 515 | mov wtemp0,1 /* Ek */ | |
| 516 | pmull2 vTmp0.1q ,vAadHash.2d,vHashKey0.2d /* auth_dat * HashKey[Total-2] */ | |
| 517 | shl vLen.2d,vLen.2d,3 /* Len */ | |
| 518 | pmull vTmp1.1q ,vAadHash.1d,vHashKey0.1d /* auth_dat * HashKey[Total-2] */ | |
| 519 | rev64 vLen.16b,vLen.16b /* Len */ | |
| 520 | ins vCtr.4s[3],wtemp0 /* Ek */ | |
| 521 | ldr qHashKey0,[hashkey_addr],16 /* auth_dat * HashKey[Total-2] */ | |
| 522 | pmull vTmp2.1q,vAadHash.1d,vHashKey0Ext.1d /* auth_dat * HashKey[Total-2] */ | |
| 523 | rev32 vEncCtr.16b,vCtr.16b /* Ek */ | |
| 524 | eor vHigh.16b,vHigh.16b,vTmp0.16b /* auth_dat * HashKey[Total-2] */ | |
| 525 | pmull2 vTmp3.1q ,vAadHash.2d,vHashKey0Ext.2d /* auth_dat * HashKey[Total-2] */ | |
| 526 | rbit vAadHash.16b,vLen.16b /* Len */ | |
| 527 | ||
| 528 | aes_encrypt_round EncCtr,Key0 /* Ek */ | |
| 529 | eor vLow.16b,vLow.16b,vTmp1.16b /* auth_dat * HashKey[Total-2] */ | |
| 530 | aes_encrypt_round EncCtr,Key1 /* Ek */ | |
| 531 | ldr qHashKey0Ext,[hashkey_addr],16 /* auth_dat * HashKey[Total-2] */ | |
| 532 | aes_encrypt_round EncCtr,Key2 /* Ek */ | |
| 533 | eor vMiddle0.16b,vMiddle0.16b,vTmp2.16b /* auth_dat * HashKey[Total-2] */ | |
| 534 | aes_encrypt_round EncCtr,Key3 /* Ek */ | |
| 535 | eor vMiddle0.16b,vMiddle0.16b,vTmp3.16b /* auth_dat * HashKey[Total-2] */ | |
| 536 | aes_encrypt_round EncCtr,Key4 /* Ek */ | |
| 537 | ||
| 538 | pmull2 vTmp0.1q,vAadHash.2d,vHashKey0.2d /* Len * HashKey[Total-1] */ | |
| 539 | pmull vTmp1.1q ,vAadHash.1d,vHashKey0.1d /* Len * HashKey[Total-1] */ | |
| 540 | aes_encrypt_round EncCtr,Key5 /* Ek */ | |
| 541 | aes_encrypt_round EncCtr,Key6 /* Ek */ | |
| 542 | pmull vTmp2.1q,vAadHash.1d,vHashKey0Ext.1d /* Len * HashKey[Total-1] */ | |
| 543 | aes_encrypt_round EncCtr,Key7 /* Ek */ | |
| 544 | eor vHigh.16b,vHigh.16b,vTmp0.16b /* Len * HashKey[Total-1] */ | |
| 545 | pmull2 vTmp3.1q ,vAadHash.2d,vHashKey0Ext.2d /* Len * HashKey[Total-1] */ | |
| 546 | aes_encrypt_round EncCtr,Key8 /* Ek */ | |
| 547 | eor vLow.16b,vLow.16b,vTmp1.16b /* Len * HashKey[Total-1] */ | |
| 548 | #if KEY_LEN==256 | |
| 549 | aes_encrypt_round EncCtr,Key9 /* Ek */ | |
| 550 | aes_encrypt_round EncCtr,Key10 /* Ek */ | |
| 551 | aes_encrypt_round EncCtr,Key11 /* Ek */ | |
| 552 | aes_encrypt_round EncCtr,Key12 /* Ek */ | |
| 553 | aese vEncCtr.16b,vKey13.16b /* Ek */ | |
| 554 | eor vEncCtr.16b,vEncCtr.16b,vKey14.16b /* Ek */ | |
| 555 | #else | |
| 556 | aese vEncCtr.16b,vKey9.16b /* Ek */ | |
| 557 | eor vEncCtr.16b,vEncCtr.16b,vKey10.16b /* Ek */ | |
| 558 | #endif | |
| 559 | eor vMiddle0.16b,vMiddle0.16b,vTmp2.16b /* Len * HashKey[Total-1] */ | |
| 560 | eor vMiddle0.16b,vMiddle0.16b,vTmp3.16b /* Len * HashKey[Total-1] */ | |
| 561 | rbit vAadHash.16b,vEncCtr.16b /* Aad */ | |
| 562 | ||
| 563 | ghash_mult_final_round AadHash,High,Low,Middle0,Tmp0,Zero,Poly | |
| 564 | ||
| 565 | ldp auth_tag,auth_tag_len,[sp,stack_size] /* Adjust here : TODO TBD */ | |
| 566 | rbit vAadHash.16b,vAadHash.16b /* Aad */ | |
| 567 | ||
| 568 | ||
| 569 | /* output auth_tag */ | |
| 570 | cmp auth_tag_len,16 | |
| 571 | bne 1f | |
| 572 | /* most likely auth_tag_len=16 */ | |
| 573 | str qAadHash,[auth_tag] | |
| 574 | pop_stack | |
| 575 | ret | |
| 576 | 1: /* auth_tag_len=12 */ | |
| 577 | cmp auth_tag_len,12 | |
| 578 | bne 1f | |
| 579 | str dAadHash,[auth_tag],8 | |
| 580 | st1 {vAadHash.s}[2],[auth_tag] | |
| 581 | pop_stack | |
| 582 | ret | |
| 583 | 1: /* auth_tag_len=8 */ | |
| 584 | str dAadHash,[auth_tag] | |
| 585 | pop_stack | |
| 586 | ret | |
| 587 | END_FUNC(dec,KEY_LEN,_) | |
| 588 | END_FUNC(dec,KEY_LEN,_nt_) |