ceph/src/crypto/isa-l/isa-l_crypto/aes/aarch64/gcm_common_128.S

   1 /**********************************************************************
   2   Copyright(c) 2021 Arm Corporation All rights reserved.
   3
   4   Redistribution and use in source and binary forms, with or without
   5   modification, are permitted provided that the following conditions
   6   are met:
   7     * Redistributions of source code must retain the above copyright
   8       notice, this list of conditions and the following disclaimer.
   9     * Redistributions in binary form must reproduce the above copyright
  10       notice, this list of conditions and the following disclaimer in
  11       the documentation and/or other materials provided with the
  12       distribution.
  13     * Neither the name of Arm Corporation nor the names of its
  14       contributors may be used to endorse or promote products derived
  15       from this software without specific prior written permission.
  16
  17   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28 **********************************************************************/
  29 #define KEY_LEN 128
  30 #include "gcm_common.S"
  31
  32 #define KEY_REGS 0,1,2,3,4,5,6,7,8
  33 .macro aes_encrypt_block    block:req
  34     aes_encrypt_round       \block,Key0
  35     aes_encrypt_round       \block,Key1
  36     aes_encrypt_round       \block,Key2
  37     aes_encrypt_round       \block,Key3
  38     aes_encrypt_round       \block,Key4
  39     aes_encrypt_round       \block,Key5
  40     aes_encrypt_round       \block,Key6
  41     aes_encrypt_round       \block,Key7
  42     aes_encrypt_round       \block,Key8
  43     aese    v\block\().16b,vKey9.16b
  44     eor     v\block\().16b,v\block\().16b,vKey10.16b
  45 .endm
  46
  47 /*
  48     Load Aes Keys to [vKey0..vKey8,vKeyLast0,vKeyLast1]
  49  */
  50 .macro load_aes_keys    key_addr:req
  51     ld1     {vKey0.4s- vKey3.4s},[\key_addr],64
  52     ld1     {vKey4.4s- vKey7.4s},[\key_addr],64
  53     ldp     qKey8,qKey9,[\key_addr],32
  54     ldr     qKey10,[\key_addr],15*16 - 128 - 32
  55 .endm
  56
  57
  58
  59 /*
  60     [low,middle,tmp0,high] +=aadhash * [hashkey0,hashkey0_ext]
  61     dat=*dat_adr
  62     enc_dat=aes_encrypt(ctr)^dat
  63     aadhash=rbit(enc_dat)
  64     [hashkey0,hashkey0_ext] = *hashkey_adr
  65     dat_adr+=16
  66     hashkey_adr+=32
  67 */
  68 .macro  aes_gcm_middle   is_enc:req,aadhash:req,dat_adr:req,hashkey_adr:req,    \
  69     hashkey0:req,hashkey0_ext:req,high:req,low:req,                             \
  70     ctr:req,enc_ctr:req,one:req,out_adr:req,                                    \
  71     tmp0:req,tmp1:req,left_count:req
  72
  73     pmull2   v\tmp0\().1q,v\aadhash\().2d,v\hashkey0\().2d
  74     pmull    v\tmp1\().1q,v\aadhash\().1d,v\hashkey0\().1d
  75     .if  \left_count > 1
  76         ldr      q\hashkey0,[\hashkey_adr],16
  77     .endif
  78
  79     add      v\ctr\().4s,v\ctr\().4s,v\one\().4s    //increase ctr
  80
  81     rev32    v\enc_ctr\().16b,v\ctr\().16b
  82     aes_encrypt_round   \enc_ctr,Key0
  83     eor      v\high\().16b,v\high\().16b,v\tmp0\().16b
  84     pmull    v\tmp0\().1q,v\aadhash\().1d,v\hashkey0_ext\().1d
  85     eor      v\low\().16b,v\low\().16b,v\tmp1\().16b
  86     pmull2   v\tmp1\().1q,v\aadhash\().2d,v\hashkey0_ext\().2d
  87     .if  \left_count > 1
  88         ldr      q\hashkey0_ext,[\hashkey_adr],16
  89     .endif
  90     eor     v\aadhash\().16b,v\aadhash\().16b,v\aadhash\().16b
  91     aes_encrypt_round   \enc_ctr,Key1
  92     aes_encrypt_round   \enc_ctr,Key2
  93     eor      v\tmp0\().16b,v\tmp1\().16b,v\tmp0\().16b
  94     aes_encrypt_round   \enc_ctr,Key3
  95     ext      v\tmp1\().16b,v\aadhash\().16b,v\tmp0\().16b,8
  96     ext      v\tmp0\().16b,v\tmp0\().16b,v\aadhash\().16b,8
  97     aes_encrypt_round   \enc_ctr,Key4
  98     eor      v\low\().16b,v\low\().16b,v\tmp1\().16b
  99     eor      v\high\().16b,v\high\().16b,v\tmp0\().16b
 100     aes_encrypt_round   \enc_ctr,Key5
 101     ldr      q\aadhash,[\dat_adr],16
 102     aes_encrypt_round   \enc_ctr,Key6
 103     aes_encrypt_round   \enc_ctr,Key7
 104     aes_encrypt_round   \enc_ctr,Key8
 105     aese    v\enc_ctr\().16b,vKey9.16b
 106     eor     v\enc_ctr\().16b,v\enc_ctr\().16b,vKey10.16b
 107     eor     v\enc_ctr\().16b,v\enc_ctr\().16b,v\aadhash\().16b
 108     .ifc \is_enc, encrypt
 109         rbit    v\aadhash\().16b,v\enc_ctr\().16b
 110     .endif
 111     .ifc \is_enc , decrypt
 112         rbit    v\aadhash\().16b,v\aadhash\().16b
 113     .endif
 114     str     q\enc_ctr,[\out_adr],16
 115 .endm
 116
 117 .macro  aes_gcm_init    is_enc:req,aadhash:req,dat_adr:req,hashkey_adr:req,     \
 118     hashkey0:req,hashkey0_ext:req, high:req,low:req,                            \
 119     ctr:req,enc_ctr:req,one:req,out_adr:req,                                    \
 120     tmp0:req,tmp1:req,left_count:req
 121     ldr     q\hashkey0,[\hashkey_adr],16
 122     add      v\ctr\().4s,v\ctr\().4s,v\one\().4s    //increase ctr
 123     rev32    v\enc_ctr\().16b,v\ctr\().16b
 124     aes_encrypt_round   \enc_ctr,Key0
 125     ldr     q\hashkey0_ext,[\hashkey_adr],16
 126     aes_encrypt_round   \enc_ctr,Key1
 127     pmull2   v\high\().1q,v\aadhash\().2d,v\hashkey0\().2d
 128     pmull    v\low\().1q,v\aadhash\().1d,v\hashkey0\().1d
 129
 130     .if  \left_count > 1
 131         ldr      q\hashkey0,[\hashkey_adr],16
 132     .endif
 133     aes_encrypt_round   \enc_ctr,Key2
 134     pmull    v\tmp1\().1q,v\aadhash\().1d,v\hashkey0_ext\().1d
 135     pmull2   v\tmp0\().1q,v\aadhash\().2d,v\hashkey0_ext\().2d
 136     eor      v\aadhash\().16b,v\aadhash\().16b,v\aadhash\().16b
 137
 138     .if  \left_count > 1
 139         ldr      q\hashkey0_ext,[\hashkey_adr],16
 140     .endif
 141     aes_encrypt_round   \enc_ctr,Key3
 142     eor      v\tmp0\().16b,v\tmp1\().16b,v\tmp0\().16b
 143
 144     aes_encrypt_round   \enc_ctr,Key4
 145     ext      v\tmp1\().16b,v\aadhash\().16b,v\tmp0\().16b,8 //low
 146     ext      v\tmp0\().16b,v\tmp0\().16b,v\aadhash\().16b,8 //high
 147     aes_encrypt_round   \enc_ctr,Key5
 148     eor      v\low\().16b,v\low\().16b,v\tmp1\().16b
 149     eor      v\high\().16b,v\high\().16b,v\tmp0\().16b
 150     aes_encrypt_round   \enc_ctr,Key6
 151     ldr      q\aadhash,[\dat_adr],16
 152     aes_encrypt_round   \enc_ctr,Key7
 153     aes_encrypt_round   \enc_ctr,Key8
 154     aese    v\enc_ctr\().16b,vKey9.16b
 155     eor     v\enc_ctr\().16b,v\enc_ctr\().16b,vKey10.16b
 156     eor     v\enc_ctr\().16b,v\enc_ctr\().16b,v\aadhash\().16b
 157     .ifc \is_enc , encrypt
 158         rbit    v\aadhash\().16b,v\enc_ctr\().16b
 159     .endif
 160     .ifc \is_enc , decrypt
 161         rbit    v\aadhash\().16b,v\aadhash\().16b
 162     .endif
 163     str     q\enc_ctr,[\out_adr],16
 164 .endm
 165