ceph/src/crypto/isa-l/isa-l_crypto/aes/aarch64/xts_aes_common.S

   1 /**********************************************************************
   2   Copyright(c) 2021 Arm Corporation All rights reserved.
   3
   4   Redistribution and use in source and binary forms, with or without
   5   modification, are permitted provided that the following conditions
   6   are met:
   7     * Redistributions of source code must retain the above copyright
   8       notice, this list of conditions and the following disclaimer.
   9     * Redistributions in binary form must reproduce the above copyright
  10       notice, this list of conditions and the following disclaimer in
  11       the documentation and/or other materials provided with the
  12       distribution.
  13     * Neither the name of Arm Corporation nor the names of its
  14       contributors may be used to endorse or promote products derived
  15       from this software without specific prior written permission.
  16
  17   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28 **********************************************************************/
  29
  30 .macro  declare_var_vector_reg name:req,reg:req
  31 .ifdef q\name
  32         .unreq  q\name
  33         .unreq  v\name
  34         .unreq  s\name
  35         .unreq  d\name
  36 .endif
  37         .set q\name , \reg
  38         q\name  .req    q\reg
  39         v\name  .req    v\reg
  40         s\name  .req    s\reg
  41         d\name  .req    d\reg
  42 .endm
  43
  44 .macro  declare_var_generic_reg name:req,reg:req
  45         \name   .req    x\reg
  46         x\name  .req    x\reg
  47         w\name  .req    w\reg
  48 .endm
  49
  50         declare_var_vector_reg  zero ,0
  51         declare_var_vector_reg  tmp,1
  52         declare_var_vector_reg  mask,2
  53         declare_var_vector_reg  dest,3
  54         declare_var_vector_reg  blk0,4
  55         declare_var_vector_reg  blk1,5
  56         declare_var_vector_reg  blk2,6
  57         declare_var_vector_reg  blk3,7
  58         declare_var_vector_reg  Key11,8
  59         declare_var_vector_reg  Key12,9
  60         declare_var_vector_reg  Key13,10
  61         declare_var_vector_reg  Key14,11
  62         declare_var_vector_reg  SavedIv,16
  63         declare_var_vector_reg  IV0,17
  64         declare_var_vector_reg  IV1,18
  65         declare_var_vector_reg  IV2,19
  66         declare_var_vector_reg  IV3,20
  67         declare_var_vector_reg  Key0,21
  68         declare_var_vector_reg  Key1,22
  69         declare_var_vector_reg  Key2,23
  70         declare_var_vector_reg  Key3,24
  71         declare_var_vector_reg  Key4,25
  72         declare_var_vector_reg  Key5,26
  73         declare_var_vector_reg  Key6,27
  74         declare_var_vector_reg  Key7,28
  75         declare_var_vector_reg  Key8,29
  76         declare_var_vector_reg  Key9,30
  77         declare_var_vector_reg  Key10,31
  78
  79 .macro aes_enc_round block:req,key:req
  80         aes_round       \block,\key,0
  81 .endm
  82
  83 .macro aes_dec_round block:req,key:req
  84         aes_round       \block,\key,1
  85 .endm
  86
  87 .macro update_iv current:req,next:req
  88         mov     ivh,\current\().d[1]
  89         mov     ivl,\current\().d[0]
  90         mov     tmpw,#0x87
  91         extr    tmpx2,ivh,ivh,#32
  92         extr    ivh,ivh,ivl,#63
  93         and     tmpw,tmpw,tmpw2,asr#31
  94         eor     ivl,tmpx,ivl,lsl#1
  95         mov     \next\().d[1],ivh
  96         mov     \next\().d[0],ivl
  97 .endm
  98
  99 .macro process_4_blks inp:req,outp:req,mode:req,is_tail
 100         update_iv       vIV0,vIV1
 101         update_iv       vIV1,vIV2
 102         ldp     qblk0,qblk1,[\inp],#32
 103         ldp     qblk2,qblk3,[\inp],#32
 104         .ifnb   \is_tail
 105                 update_iv       vIV2, vSavedIv
 106                 update_iv       vSavedIv,vIV3
 107         .else
 108                 update_iv       vIV2,vIV3
 109         .endif
 110         eor     vblk0.16b,vblk0.16b,vIV0.16b
 111         eor     vblk1.16b,vblk1.16b,vIV1.16b
 112         eor     vblk2.16b,vblk2.16b,vIV2.16b
 113         eor     vblk3.16b,vblk3.16b,vIV3.16b
 114
 115         aes_rounds_interleave vblk0,vblk1,vblk2,vblk3,\mode
 116         eor     vblk0.16b,vblk0.16b,vIV0.16b
 117         eor     vblk1.16b,vblk1.16b,vIV1.16b
 118         stp     qblk0,qblk1,[\outp],#32
 119         eor     vblk2.16b,vblk2.16b,vIV2.16b
 120         eor     vblk3.16b,vblk3.16b,vIV3.16b
 121         stp     qblk2,qblk3,[\outp],#32
 122         .ifb \is_tail
 123                 update_iv       vIV3,vIV0
 124         .endif
 125 .endm
 126
 127 .macro process_1_blk inp:req,outp:req,mode:req
 128         ld1     {vblk0.16b},[\inp],#16
 129         eor     vblk0.16b,vblk0.16b,vIV0.16b
 130         aes_rounds      vblk0,\mode
 131         eor     vblk0.16b,vblk0.16b,vIV0.16b
 132         str     qblk0,[\outp], #16
 133 .endm
 134
 135         key2    .req    x0
 136         key1    .req    x1
 137         iv      .req    x2
 138         bytes   .req    x3
 139         inp     .req    x4
 140         outp    .req    x5
 141         rcon    .req    w6
 142         blocks  .req    x7
 143         tmpx    .req    x8
 144         tmpw    .req    w8
 145         tmpw2   .req    w9
 146         tmpx2   .req    x9
 147         ivl     .req    x10
 148         ivh     .req    x11
 149         lastblk .req    x12
 150         tmpbuf  .req    x13
 151         tailcnt .req    x14
 152         rcon2   .req    w15
 153
 154 .macro xts_aes_crypt mode:req,expander,more:vararg
 155         save_stack
 156
 157         ld1     {vIV0.16b},[iv],16
 158         .ifnb \expander
 159                 \expander\()    \more
 160         .endif
 161         lsr     blocks,bytes,4
 162         and     tailcnt,bytes,#0x0F
 163
 164         cmp     bytes,16
 165         b.lt    .return
 166
 167 .process_4_blks:
 168         cmp     blocks, 4
 169         b.lt    .singles
 170         subs    blocks,blocks,4
 171         /* in decryption mode, check whether this is
 172          * last block before the less-than-one-block tail
 173          * need to swap tweak in this case
 174          */
 175         .if \mode == 1
 176                 b.gt    .not_tail_4blk
 177                 cmp     tailcnt,1
 178                 b.lt    .not_tail_4blk
 179                 process_4_blks  inp,outp,\mode,1
 180                 b       .process_4_blks
 181 .not_tail_4blk:
 182         .endif
 183         process_4_blks  inp,outp,\mode
 184         b       .process_4_blks
 185
 186 .singles:
 187         subs    blocks,blocks,#1
 188         b.lt    .checktail
 189         /* in decryption mode, check whether this is
 190          *last block before the less-than-one-block tail
 191          * need to swap tweak in this case
 192          */
 193         .if \mode == 1
 194                 b.gt    .not_tail_1blk
 195                 cmp     tailcnt,1
 196                 b.lt    .not_tail_1blk
 197                 mov     vSavedIv.16b, vIV0.16b
 198                 update_iv       vSavedIv, vIV0
 199                 process_1_blk   inp,outp,\mode
 200                 b       .checktail
 201 .not_tail_1blk:
 202         .endif
 203         process_1_blk   inp,outp,\mode
 204         update_iv       vIV0,vIV0
 205         b .singles
 206 .checktail:
 207         cmp     tailcnt,1
 208         b.lt    .return
 209         sub     lastblk,outp,#16
 210 .copytail:
 211         subs    tailcnt,tailcnt,#1
 212         ldrb    tmpw,[lastblk,tailcnt]
 213         strb    tmpw,[outp,tailcnt]
 214         ldrb    tmpw,[inp,tailcnt]
 215         strb    tmpw,[tmpbuf,tailcnt]
 216         b.gt    .copytail
 217         and     tailcnt,bytes,#0x0F
 218 .steal:
 219         cmp     tailcnt,15
 220         ldrb    tmpw,[lastblk,tailcnt]
 221         strb    tmpw,[tmpbuf,tailcnt]
 222         add     tailcnt,tailcnt,#1
 223         b.lt    .steal
 224         .if \mode == 1
 225                 mov     vIV0.16b,vSavedIv.16b
 226         .endif
 227         process_1_blk   tmpbuf,lastblk,\mode
 228 .return:
 229         restore_stack
 230         ret
 231 .endm
 232