ceph/src/crypto/isa-l/isa-l_crypto/aes/aarch64/gcm_common.S

   1 /**********************************************************************
   2   Copyright(c) 2021 Arm Corporation All rights reserved.
   3
   4   Redistribution and use in source and binary forms, with or without
   5   modification, are permitted provided that the following conditions
   6   are met:
   7     * Redistributions of source code must retain the above copyright
   8       notice, this list of conditions and the following disclaimer.
   9     * Redistributions in binary form must reproduce the above copyright
  10       notice, this list of conditions and the following disclaimer in
  11       the documentation and/or other materials provided with the
  12       distribution.
  13     * Neither the name of Arm Corporation nor the names of its
  14       contributors may be used to endorse or promote products derived
  15       from this software without specific prior written permission.
  16
  17   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28 **********************************************************************/
  29     .arch armv8-a+crypto
  30     .text
  31 #define HASHKEY_TOTAL_NUM       (24)
  32 #define HASHKEY_BASE_OFF        (15*16)
  33 #define HASHKEY_OFF(n)          ((15*16)+n*32)
  34 #define HASHKEY_EXT_OFF(n)      ((15*16)+n*32+16)
  35 #ifndef KEY_LEN
  36 #define KEY_LEN 128
  37 #endif
  38 #ifndef BLOCKS
  39 #define BLOCKS 24
  40 #endif
  41 #define FN_NAME(fn,mode,post) aes_gcm_##fn##_##mode####post##aes
  42 #define START_FUNC(fn,mode,post) .global FN_NAME(fn,mode,post); \
  43     .type   FN_NAME(fn,mode,post), %function; \
  44     FN_NAME(fn,mode,post):
  45 #define END_FUNC(fn,mode,post)   .size  FN_NAME(fn,mode,post), .-FN_NAME(fn,mode,post)
  46
  47 #define AAD_LEN_OFF                 16
  48 #define IN_LENGTH_OFF               24
  49 #define PARTIAL_BLOCK_ENC_KEY_OFF   32
  50 #define PARTIAL_BLOCK_LENGTH_OFF    80
  51 #define CTR_OFF                     64
  52 #define ORIG_IV_OFF                 48
  53 /*
  54     [low,middle,tmp0,high] +=dat0 * [hashkey0,hashkey0_ext]
  55     ifnb dat1
  56         dat1=rbit(*dat_adr)
  57         [hashkey0,hashkey0_ext] = *hashkey_adr
  58         dat_adr+=16
  59         hashkey_adr+=32
  60 */
  61
  62 .macro  ghash_mult_round    aadhash:req,dat_adr:req,hashkey_adr:req,   \
  63     hashkey0:req,hashkey0_ext:req,high:req,low:req,middle:req,         \
  64     tmp0:req,tmp1:req,next_dat:req,left_count:req
  65
  66     ldr      q\next_dat,[\dat_adr],16
  67     pmull    v\tmp0\().1q,v\aadhash\().1d,v\hashkey0_ext\().1d
  68     pmull2   v\tmp1\().1q,v\aadhash\().2d,v\hashkey0_ext\().2d
  69     .if  \left_count > 1
  70         ldr      q\hashkey0_ext,[\hashkey_adr,16]
  71     .endif
  72     eor      v\middle\().16b,v\middle\().16b,v\tmp0\().16b
  73     pmull2   v\tmp0\().1q,v\aadhash\().2d,v\hashkey0\().2d
  74     eor      v\middle\().16b,v\middle\().16b,v\tmp1\().16b
  75     pmull    v\tmp1\().1q,v\aadhash\().1d,v\hashkey0\().1d
  76     .if  \left_count > 1
  77         ldr      q\hashkey0,[\hashkey_adr],32
  78     .endif
  79     eor      v\high\().16b,v\high\().16b,v\tmp0\().16b
  80     eor      v\low\().16b,v\low\().16b,v\tmp1\().16b
  81     rbit     v\aadhash\().16b, v\next_dat\().16b
  82 .endm
  83
  84 .macro  ghash_mult_init_round    aadhash:req,dat_adr:req,hashkey_adr:req,   \
  85     hashkey0:req,hashkey0_ext:req,                                          \
  86     high:req,low:req,middle:req,tmp0:req,next_dat:req,left_count:req
  87     ldp     q\hashkey0,q\hashkey0_ext,[\hashkey_adr],32
  88     ldr      q\next_dat,[\dat_adr],16
  89     pmull    v\middle\().1q,v\aadhash\().1d,v\hashkey0_ext\().1d
  90     pmull2   v\tmp0\().1q,v\aadhash\().2d,v\hashkey0_ext\().2d
  91     .if  \left_count > 1
  92         ldr      q\hashkey0_ext,[\hashkey_adr,16]
  93     .endif
  94     pmull2   v\high\().1q,v\aadhash\().2d,v\hashkey0\().2d
  95     eor      v\middle\().16b,v\middle\().16b,v\tmp0\().16b
  96
  97     pmull    v\low\().1q,v\aadhash\().1d,v\hashkey0\().1d
  98     .if  \left_count > 1
  99         ldr      q\hashkey0,[\hashkey_adr],32
 100     .endif
 101     rbit     v\aadhash\().16b, v\next_dat\().16b
 102 .endm
 103
 104 /* aadhash=reduction(low,middle,high)+dat0 */
 105 .macro  ghash_mult_final_round aadhash:req,     \
 106     high:req,low:req,middle:req,tmp0:req,       \
 107     zero:req,poly:req
 108
 109     ext      v\tmp0\().16b,v\middle\().16b,v\zero\().16b,8      /*high*/
 110     ext      v\middle\().16b,v\zero\().16b,v\middle\().16b,8    /*low */
 111     eor      v\high\().16b,v\high\().16b,v\tmp0\().16b
 112     eor      v\low\().16b,v\low\().16b,v\middle\().16b
 113
 114     pmull2   v\middle\().1q,v\high\().2d,v\poly\().2d
 115
 116     ext      v\tmp0\().16b,v\middle\().16b,v\zero\().16b,8      /*high*/
 117     ext      v\middle\().16b,v\zero\().16b,v\middle\().16b,8    /*low*/
 118     eor      v\high\().16b,v\high\().16b,v\tmp0\().16b
 119     eor      v\low\().16b,v\low\().16b,v\middle\().16b
 120     pmull    v\middle\().1q,v\high\().1d,v\poly\().1d
 121     eor      v\tmp0\().16b, v\low\().16b, v\middle\().16b
 122     eor      v\aadhash\().16b, v\aadhash\().16b, v\tmp0\().16b
 123 .endm
 124 .macro  ghash_reset_hashkey_addr    hashkey_addr:req,hashkey_base:req,count:req
 125     add     \hashkey_addr,\hashkey_base,(24-\count)<<5
 126 .endm
 127
 128
 129 .macro ghash_block_n count:req,aadhash:req, dat:req,dat_addr:req, hashkey_addr:req, hashkey_base:req, \
 130     hashkey:req,hashkey_ext:req,high:req,low:req,middle:req, zero:req,poly:req, \
 131     tmp0:req,tmp1:req
 132
 133     ghash_reset_hashkey_addr    \hashkey_addr,\hashkey_base,\count
 134     ghash_mult_init_round   \aadhash,\dat_addr,\hashkey_addr,\hashkey,\hashkey_ext, \
 135         \high,\low,\middle,\tmp0,\dat,\count
 136     .set left_count,\count - 1
 137     .rept left_count
 138         ghash_mult_round        \aadhash,\dat_addr,\hashkey_addr,\hashkey,\hashkey_ext, \
 139             \high,\low,\middle,\tmp0,\tmp1,\dat, left_count
 140         .set left_count,left_count - 1
 141
 142     .endr
 143     ghash_mult_final_round  \aadhash,\high,\low,\middle,\tmp0,\zero,\poly
 144 .endm
 145
 146 /*
 147     aadhash=aadhash*[hashkey,hashkey_ext] + rbit(dat)
 148 */
 149 .macro ghash_block_reg aadhash:req, dat:req, \
 150     hashkey:req,hashkey_ext:req,high:req,low:req,middle:req, zero:req,poly:req, \
 151     tmp0:req
 152     pmull    v\middle\().1q,v\aadhash\().1d,v\hashkey_ext\().1d
 153     pmull2   v\tmp0\().1q,v\aadhash\().2d,v\hashkey_ext\().2d
 154     pmull2   v\high\().1q,v\aadhash\().2d,v\hashkey\().2d
 155     eor      v\middle\().16b,v\middle\().16b,v\tmp0\().16b
 156     pmull    v\low\().1q,v\aadhash\().1d,v\hashkey\().1d
 157     rbit     v\aadhash\().16b, v\dat\().16b
 158     ghash_mult_final_round  \aadhash,\high,\low,\middle,\tmp0,\zero,\poly
 159 .endm
 160
 161 .macro  ghash_mult_round_noload    aadhash:req,   \
 162     hashkey0:req,hashkey0_ext:req,high:req,low:req,middle:req,         \
 163     tmp0:req,tmp1:req
 164
 165     pmull    v\tmp0\().1q,v\aadhash\().1d,v\hashkey0_ext\().1d
 166     pmull2   v\tmp1\().1q,v\aadhash\().2d,v\hashkey0_ext\().2d
 167     eor      v\middle\().16b,v\middle\().16b,v\tmp0\().16b
 168     pmull2   v\tmp0\().1q,v\aadhash\().2d,v\hashkey0\().2d
 169     eor      v\middle\().16b,v\middle\().16b,v\tmp1\().16b
 170     pmull    v\tmp1\().1q,v\aadhash\().1d,v\hashkey0\().1d
 171     eor      v\high\().16b,v\high\().16b,v\tmp0\().16b
 172     eor      v\low\().16b,v\low\().16b,v\tmp1\().16b
 173
 174 .endm
 175
 176 /* aadhash=reduction([low,high],poly)+dat0 */
 177 .macro  poly_mult_final_x2 aadhash:req, \
 178     high:req,low:req,tmp0:req,tmp1:req,      \
 179     poly:req
 180     pmull2   v\tmp1\().1q,v\high\().2d,v\poly\().2d
 181     eor      v\low\().16b, v\aadhash\().16b, v\low\().16b
 182     eor      v\aadhash\().16b,v\aadhash\().16b,v\aadhash\().16b
 183     ext      v\tmp0\().16b,v\tmp1\().16b,v\aadhash\().16b,8 //high
 184     ext      v\tmp1\().16b,v\aadhash\().16b,v\tmp1\().16b,8 //low
 185     eor      v\high\().16b,v\high\().16b,v\tmp0\().16b
 186     eor      v\low\().16b,v\low\().16b,v\tmp1\().16b
 187     pmull    v\tmp1\().1q,v\high\().1d,v\poly\().1d
 188     eor      v\aadhash\().16b, v\low\().16b, v\tmp1\().16b
 189 .endm
 190
 191 .macro aes_encrypt_round    block,key
 192     aese    v\block\().16b,v\key\().16b
 193     aesmc   v\block\().16b,v\block\().16b
 194 .endm
 195
 196 .macro  declare_var_vector_reg name:req,reg:req
 197     q\name      .req    q\reg
 198     v\name      .req    v\reg
 199     s\name      .req    s\reg
 200     d\name      .req    d\reg
 201 .endm
 202
 203 .macro  declare_var_generic_reg name:req,reg:req
 204      \name      .req    x\reg
 205     x\name      .req    x\reg
 206     w\name      .req    w\reg
 207 .endm
 208
 209 /*Read data less than 16 */
 210 .macro read_small_data  dest:req,src:req,size:req,tbl_adr:req,tbl:req
 211     ldr     q\tbl,[\tbl_adr,\size,lsl 4]
 212     tbz     \size,3,1f
 213     ld1     {v\dest\().d}[0],[\src],8
 214 1:
 215     tbz     \size,2,1f
 216     ld1     {v\dest\().s}[2],[\src],4
 217 1:
 218     tbz     \size,1,1f
 219     ld1     {v\dest\().h}[6],[\src],2
 220 1:
 221     tbz     \size,0,1f
 222     ld1     {v\dest\().b}[14],[\src],1
 223 1:
 224     tbl     v\dest\().16b,{v\dest\().16b},v\tbl\().16b
 225 .endm
 226 .macro read_small_data_start  dest:req,src:req,size:req,tbl_adr:req,tbl:req
 227     adrp    \tbl_adr,:got:read_small_data_table
 228     ldr     \tbl_adr,[\tbl_adr,#:got_lo12:read_small_data_table]
 229     read_small_data \dest,\src,\size,\tbl_adr,\tbl
 230 .endm
 231
 232 .macro read_small_data_end  dest:req,src:req,size:req,tbl_adr:req,tbl:req
 233     adrp    \tbl_adr,:got:read_end_small_data_table
 234     ldr     \tbl_adr,[\tbl_adr,#:got_lo12:read_end_small_data_table]
 235     read_small_data \dest,\src,\size,\tbl_adr,\tbl
 236 .endm
 237
 238 .macro write_small_data  src:req,dest:req,size:req,tbl_adr:req,tmp1:req
 239     ldr     q\tmp1,[\tbl_adr,\size,lsl 4]
 240     tbl     v\tmp1\().16b,{v\src\().16b},v\tmp1\().16b
 241     tbz     \size,3,1f
 242     st1     {v\tmp1\().d}[0],[\dest],8
 243 1:
 244     tbz     \size,2,1f
 245     st1     {v\tmp1\().s}[2],[\dest],4
 246 1:
 247     tbz     \size,1,1f
 248     st1     {v\tmp1\().h}[6],[\dest],2
 249 1:
 250     tbz     \size,0,1f
 251     st1     {v\tmp1\().b}[14],[\dest],1
 252 1:
 253 .endm
 254 .macro write_small_data_start  src:req,dest:req,size:req,tbl_adr:req,tmp1:req
 255     adrp    \tbl_adr,:got:write_small_data_table
 256     ldr     \tbl_adr,[\tbl_adr,#:got_lo12:write_small_data_table]
 257     write_small_data \src,\dest,\size,\tbl_adr,\tmp1
 258 .endm
 259 .macro write_small_data_end  src:req,dest:req,size:req,tbl_adr:req,tmp1:req
 260     adrp    \tbl_adr,:got:write_end_small_data_table
 261     ldr     \tbl_adr,[\tbl_adr,#:got_lo12:write_end_small_data_table]
 262     write_small_data \src,\dest,\size,\tbl_adr,\tmp1
 263 .endm
 264
 265 .macro tbx_small_data_end   src:req,dest:req,size:req,tbl_adr:req,tmp1:req
 266     adrp    \tbl_adr,:got:tbx_end_small_data_table
 267     ldr     \tbl_adr,[\tbl_adr,#:got_lo12:tbx_end_small_data_table]
 268     ldr     q\tmp1,[\tbl_adr,\size,lsl 4]
 269     tbx     v\dest\().16b,{v\src\().16b},v\tmp1\().16b
 270 .endm
 271
 272 .macro tbx_small_data_start   src:req,dest:req,size:req,tbl_adr:req,tmp1:req
 273     adrp    \tbl_adr,:got:tbx_start_small_data_table
 274     ldr     \tbl_adr,[\tbl_adr,#:got_lo12:tbx_start_small_data_table]
 275     ldr     q\tmp1,[\tbl_adr,\size,lsl 4]
 276     tbx     v\dest\().16b,{v\src\().16b},v\tmp1\().16b
 277 .endm
 278
 279
 280 .macro clear_small_data   dest:req,zero:req,size:req,tbl_adr:req,tmp1:req
 281     adrp    \tbl_adr,:got:shift_small_data_table
 282     ldr     \tbl_adr,[\tbl_adr,#:got_lo12:shift_small_data_table]
 283     add     \tbl_adr,\tbl_adr,16
 284     sub     \tbl_adr,\tbl_adr,\size
 285     ldr     q\tmp1,[\tbl_adr]
 286     tbx     v\dest\().16b,{v\zero\().16b},v\tmp1\().16b
 287 .endm
 288
 289
 290 .macro aes_gcm_n_round is_enc:req,count:req,aadhash:req, dat_addr:req,  \
 291     hashkey_addr:req, hashkey_base:req,                                 \
 292     hashkey:req,hashkey_ext:req,high:req,low:req, poly:req,             \
 293     ctr:req,enc_ctr:req,one:req,out_adr:req,                            \
 294     tmp0:req,tmp1:req
 295
 296     ghash_reset_hashkey_addr    \hashkey_addr,\hashkey_base,\count
 297
 298     aes_gcm_init    \is_enc,\aadhash,\dat_addr,\hashkey_addr,           \
 299         \hashkey,\hashkey_ext, \high,\low,                              \
 300         \ctr,\enc_ctr,\one,\out_adr,                                    \
 301         \tmp0,\tmp1,\count
 302
 303     .set left_count,\count - 1
 304     .rept left_count
 305         aes_gcm_middle        \is_enc,\aadhash,\dat_addr,\hashkey_addr, \
 306             \hashkey,\hashkey_ext, \high,\low,                          \
 307             \ctr,\enc_ctr,\one,\out_adr,                                \
 308             \tmp0,\tmp1, left_count
 309         .set left_count,left_count - 1
 310     .endr
 311
 312     poly_mult_final_x2  \aadhash,\high,\low,\tmp0,\tmp1,\poly
 313
 314 .endm
 315
 316
 317 /*
 318     aadhash=aadhash*[hashkey_base[(TOTAL_HASHKEY_NUM-2),(TOTAL_HASHKEY_NUM-1)]] + rbit(dat)
 319 */
 320 .macro ghash_block_reg_x2 aadhash:req, dat:req, hashkey_base:req,       \
 321     hashkey:req,high:req,low:req,tmp0:req, tmp1:req,                    \
 322     tmp2:req,temp0:req
 323     ldr     q\hashkey,[\hashkey_base,(TOTAL_HASHKEY_NUM-1)*32+16]
 324     eor     v\tmp2\().16b,v\tmp2\().16b,v\tmp2\().16b,8 //zero
 325     pmull    v\tmp1\().1q,v\aadhash\().1d,v\hashkey\().1d
 326     pmull2   v\tmp0\().1q,v\aadhash\().2d,v\hashkey\().2d
 327     ldr     q\hashkey,[\hashkey_base,(TOTAL_HASHKEY_NUM-1)*32]
 328     eor      v\tmp0\().16b,v\tmp1\().16b,v\tmp0\().16b
 329     ext      v\tmp0\().16b,v\tmp0\().16b,v\tmp2\().16b,8 /*high*/
 330     ext      v\tmp1\().16b,v\tmp2\().16b,v\tmp0\().16b,8 /*low*/
 331     pmull2   v\high\().1q,v\aadhash\().2d,v\hashkey\().2d
 332     mov      temp0,0x87
 333     pmull    v\low\().1q,v\aadhash\().1d,v\hashkey\().1d
 334     dup      v\tmp2\().2d,x0
 335     eor      v\high\().16b,v\high\().16b,v\tmp0\().16b
 336     eor      v\low\().16b,v\low\().16b,v\tmp1\().16b
 337     rbit     v\aadhash\().16b, v\dat\().16b
 338     poly_mult_final_x2  \aadhash,\high,\low,\tmp0,\tmp1,\tmp2
 339 .endm
 340
 341 .macro __generic_load_small_data  is_enc:req,len_bit:req,small_read_len:req,  \
 342         in_adr:req,out_adr:req,partial_block:req,temp0:req,temp1:req,r:req,p
 343     tbz         \small_read_len,\len_bit,1f
 344     ldr\p       \r\()\temp0,[\in_adr],1<<\len_bit           /*in */
 345     ldr\p       \r\()\temp1,[\partial_block]                /* partial*/
 346     eor         \r\()\temp1,\r\()\temp0,\r\()\temp1
 347     .ifc \is_enc ,decrypt
 348         str\p   \r\()\temp0,[\partial_block],1<<\len_bit
 349     .endif
 350     .ifc \is_enc, encrypt
 351         str\p   \r\()\temp1,[\partial_block],1<<\len_bit
 352     .endif
 353     str\p   \r\()\temp1,[\out_adr],1<<\len_bit
 354 1:
 355 .endm
 356 .macro  generic_load_partial_block is_enc:req,small_read_len:req,in_adr:req,out_adr:req, \
 357     partial_block:req,temp0:req,temp1:req
 358     __generic_load_small_data \is_enc,3,\small_read_len,\in_adr,\out_adr,\partial_block,\temp0,\temp1,x     /*  small_read_len >=8 */
 359     __generic_load_small_data \is_enc,2,\small_read_len,\in_adr,\out_adr,\partial_block,\temp0,\temp1,w     /*  small_read_len >=4 */
 360     __generic_load_small_data \is_enc,1,\small_read_len,\in_adr,\out_adr,\partial_block,\temp0,\temp1,w,h   /*  small_read_len >=2 */
 361     __generic_load_small_data \is_enc,0,\small_read_len,\in_adr,\out_adr,\partial_block,\temp0,\temp1,w,b   /*  small_read_len >=1 */
 362 .endm
 363 /* without Neon read version */
 364 .macro  generic_partial_block_start    is_enc:req,in_len:req,in_adr:req,out_adr:req,context:req,   \
 365         partial_block:req,partial_block_len:req,small_read_len:req,left_partial_block_len:req,     \
 366         temp0:req
 367     mov         \left_partial_block_len,16
 368     add         \partial_block,\context,PARTIAL_BLOCK_ENC_KEY_OFF
 369     sub         \left_partial_block_len,\left_partial_block_len,\partial_block_len
 370     add         \partial_block,\partial_block,\partial_block_len
 371     cmp         \in_len,\left_partial_block_len
 372     csel        \small_read_len,\in_len,\left_partial_block_len, ls
 373     add         \partial_block_len,\partial_block_len,\small_read_len
 374     sub         \in_len,\in_len,\small_read_len
 375     and         \partial_block_len,\partial_block_len,0xf
 376     str         \partial_block_len,[\context,PARTIAL_BLOCK_LENGTH_OFF]
 377     generic_load_partial_block \is_enc,\small_read_len,\in_adr,\out_adr,\partial_block, \
 378         \left_partial_block_len,\temp0     /*  small_read_len >=8 */
 379 .endm
 380 .macro  generic_paritial_block_end    is_enc:req,in_len:req,in_adr:req,out_adr:req,context:req,   \
 381         partial_block:req,temp0:req,temp1:req
 382     str         \in_len,[\context,PARTIAL_BLOCK_LENGTH_OFF]
 383     add         \partial_block,\context,PARTIAL_BLOCK_ENC_KEY_OFF
 384     generic_load_partial_block \is_enc,\in_len,\in_adr,\out_adr,\partial_block,\temp0,\temp1     /*  small_read_len >=8 */
 385 .endm
 386 /*partial_block_len+in_len < 16,partial_block_len=0,in_len>0 */
 387 .macro  paritial_block_small_length is_enc:req,context:req,in_len:req,in_adr:req,out_adr:req,temp0:req,temp1:req,Ctr:req
 388
 389     cbz     1f
 390     ldr     \temp0,[\context,PARTIAL_BLOCK_LENGTH_OFF]
 391     add     \temp1,\temp0,\in_len
 392     str     \temp1,[\context,PARTIAL_BLOCK_LENGTH_OFF]
 393     add     \context,\temp0,PARTIAL_BLOCK_ENC_KEY_OFF
 394 2:/* loop start */
 395     sub     \in_len,\in_len,1
 396     ldrb    w\temp0,[\in_adr],1
 397     ldrb    w\temp1,[\context]
 398     eor     w\temp1,w\temp1,w\temp0
 399     strb    w\temp1,[\out_adr],1
 400 .ifc \is_enc , encrypt
 401     strb    w\temp1,[\context],1
 402 .endif
 403 .ifc \is_enc,decrypt
 404     strb    w\temp0,[\context],1
 405 .endif
 406     cbnz     \in_len,2b
 407 1:/* loop end */
 408 .endm
 409
 410 /* 0<in_len < 16,partial_block_len=0 */
 411 .macro  paritial_block_end  is_enc:req,context:req,in_len:req,in_adr:req,out_adr:req,   \
 412     temp0:req,partial_block_len:req    \
 413     PartialBlock:req,ctr:req,one:req,Tmp2:req,Tmp3:req,Tmp4:req
 414     add         v\ctr\().4s,v\ctr\().4s,v\one\().4s    //increase ctr
 415     str         q\ctr,[context,CTR_OFF]
 416     read_small_data_start   \PartialBlock,\in_adr,\in_len,\tbl_adr,\Tmp0
 417     aes_encrypt_block   \ctr
 418
 419 .endm
 420         declare_var_vector_reg  Key0 ,16
 421         declare_var_vector_reg  Key1 ,17
 422         declare_var_vector_reg  Key2 ,18
 423         declare_var_vector_reg  Key3 ,19
 424         declare_var_vector_reg  Key4 ,20
 425         declare_var_vector_reg  Key5 ,21
 426         declare_var_vector_reg  Key6 ,22
 427         declare_var_vector_reg  Key7 ,23
 428         declare_var_vector_reg  Key8 ,24
 429         declare_var_vector_reg  Key9 ,25
 430         declare_var_vector_reg  Key10,26