ceph/src/crypto/isa-l/isa-l_crypto/aes/aarch64/gcm_common_256.S

   1 /**********************************************************************
   2   Copyright(c) 2021 Arm Corporation All rights reserved.
   3
   4   Redistribution and use in source and binary forms, with or without
   5   modification, are permitted provided that the following conditions
   6   are met:
   7     * Redistributions of source code must retain the above copyright
   8       notice, this list of conditions and the following disclaimer.
   9     * Redistributions in binary form must reproduce the above copyright
  10       notice, this list of conditions and the following disclaimer in
  11       the documentation and/or other materials provided with the
  12       distribution.
  13     * Neither the name of Arm Corporation nor the names of its
  14       contributors may be used to endorse or promote products derived
  15       from this software without specific prior written permission.
  16
  17   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  18   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  19   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  20   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  21   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  22   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  23   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  24   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  25   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  26   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  27   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  28 **********************************************************************/
  29 #define KEY_LEN 256
  30 #include "gcm_common.S"
  31         declare_var_vector_reg  Key11,27
  32         declare_var_vector_reg  Key12,28
  33         declare_var_vector_reg  Key13,29
  34         declare_var_vector_reg  Key14,30
  35 #define KEY_REGS 0,1,2,3,4,5,6,7,8,9,10,11,12
  36 .macro aes_encrypt_block    block:req
  37     aes_encrypt_round       \block,Key0
  38     aes_encrypt_round       \block,Key1
  39     aes_encrypt_round       \block,Key2
  40     aes_encrypt_round       \block,Key3
  41     aes_encrypt_round       \block,Key4
  42     aes_encrypt_round       \block,Key5
  43     aes_encrypt_round       \block,Key6
  44     aes_encrypt_round       \block,Key7
  45     aes_encrypt_round       \block,Key8
  46     aes_encrypt_round       \block,Key9
  47     aes_encrypt_round       \block,Key10
  48     aes_encrypt_round       \block,Key11
  49     aes_encrypt_round       \block,Key12
  50     aese    v\block\().16b,vKey13.16b
  51     eor     v\block\().16b,v\block\().16b,vKey14.16b
  52 .endm
  53
  54 /*
  55     Load Aes Keys to [vKey0..vKey8,vKeyLast0,vKeyLast1]
  56  */
  57 .macro load_aes_keys    key_addr:req
  58     ld1     { vKey0.4s-  vKey3.4s},[\key_addr],64
  59     ld1     { vKey4.4s-  vKey7.4s},[\key_addr],64
  60     ld1     { vKey8.4s- vKey11.4s},[\key_addr],64
  61     ld1     {vKey12.4s- vKey14.4s},[\key_addr],48
  62 .endm
  63
  64
  65
  66 /*
  67     [low,middle,tmp0,high] +=aadhash * [hashkey0,hashkey0_ext]
  68     dat=*dat_adr
  69     enc_dat=aes_encrypt(ctr)^dat
  70     aadhash=rbit(enc_dat)
  71     [hashkey0,hashkey0_ext] = *hashkey_adr
  72     dat_adr+=16
  73     hashkey_adr+=32
  74 */
  75 .macro  aes_gcm_middle   is_enc:req,aadhash:req,dat_adr:req,hashkey_adr:req,    \
  76     hashkey0:req,hashkey0_ext:req,high:req,low:req,                             \
  77     ctr:req,enc_ctr:req,one:req,out_adr:req,                                    \
  78     tmp0:req,tmp1:req,left_count:req
  79
  80     pmull2   v\tmp0\().1q,v\aadhash\().2d,v\hashkey0\().2d
  81     pmull    v\tmp1\().1q,v\aadhash\().1d,v\hashkey0\().1d
  82     .if  \left_count > 1
  83         ldr      q\hashkey0,[\hashkey_adr],16
  84     .endif
  85
  86     add      v\ctr\().4s,v\ctr\().4s,v\one\().4s    //increase ctr
  87
  88     rev32    v\enc_ctr\().16b,v\ctr\().16b
  89     aes_encrypt_round   \enc_ctr,Key0
  90     eor      v\high\().16b,v\high\().16b,v\tmp0\().16b
  91     pmull    v\tmp0\().1q,v\aadhash\().1d,v\hashkey0_ext\().1d
  92     eor      v\low\().16b,v\low\().16b,v\tmp1\().16b
  93     pmull2   v\tmp1\().1q,v\aadhash\().2d,v\hashkey0_ext\().2d
  94     .if  \left_count > 1
  95         ldr      q\hashkey0_ext,[\hashkey_adr],16
  96     .endif
  97     eor     v\aadhash\().16b,v\aadhash\().16b,v\aadhash\().16b
  98     aes_encrypt_round   \enc_ctr,Key1
  99     aes_encrypt_round   \enc_ctr,Key2
 100     eor      v\tmp0\().16b,v\tmp1\().16b,v\tmp0\().16b
 101     aes_encrypt_round   \enc_ctr,Key3
 102     ext      v\tmp1\().16b,v\aadhash\().16b,v\tmp0\().16b,8
 103     ext      v\tmp0\().16b,v\tmp0\().16b,v\aadhash\().16b,8
 104     aes_encrypt_round   \enc_ctr,Key4
 105     eor      v\low\().16b,v\low\().16b,v\tmp1\().16b
 106     eor      v\high\().16b,v\high\().16b,v\tmp0\().16b
 107     aes_encrypt_round   \enc_ctr,Key5
 108     ldr      q\aadhash,[\dat_adr],16
 109     aes_encrypt_round   \enc_ctr,Key6
 110     aes_encrypt_round   \enc_ctr,Key7
 111     aes_encrypt_round   \enc_ctr,Key8
 112     aes_encrypt_round   \enc_ctr,Key9
 113     aes_encrypt_round   \enc_ctr,Key10
 114     aes_encrypt_round   \enc_ctr,Key11
 115     aes_encrypt_round   \enc_ctr,Key12
 116     aese    v\enc_ctr\().16b,vKey13.16b
 117     eor     v\enc_ctr\().16b,v\enc_ctr\().16b,vKey14.16b
 118     eor     v\enc_ctr\().16b,v\enc_ctr\().16b,v\aadhash\().16b
 119     .ifc \is_enc , encrypt
 120         rbit    v\aadhash\().16b,v\enc_ctr\().16b
 121     .endif
 122     .ifc \is_enc , decrypt
 123         rbit    v\aadhash\().16b,v\aadhash\().16b
 124     .endif
 125     str     q\enc_ctr,[\out_adr],16
 126 .endm
 127
 128 .macro  aes_gcm_init    is_enc:req,aadhash:req,dat_adr:req,hashkey_adr:req,   \
 129     hashkey0:req,hashkey0_ext:req, high:req,low:req,                        \
 130     ctr:req,enc_ctr:req,one:req,out_adr:req,                                \
 131     tmp0:req,tmp1:req,left_count:req
 132     ldr     q\hashkey0,[\hashkey_adr],16
 133     add      v\ctr\().4s,v\ctr\().4s,v\one\().4s    /*increase ctr */
 134     rev32    v\enc_ctr\().16b,v\ctr\().16b
 135     aes_encrypt_round   \enc_ctr,Key0
 136     ldr     q\hashkey0_ext,[\hashkey_adr],16
 137     aes_encrypt_round   \enc_ctr,Key1
 138     pmull2   v\high\().1q,v\aadhash\().2d,v\hashkey0\().2d
 139     pmull    v\low\().1q,v\aadhash\().1d,v\hashkey0\().1d
 140
 141     .if  \left_count > 1
 142         ldr      q\hashkey0,[\hashkey_adr],16
 143     .endif
 144     aes_encrypt_round   \enc_ctr,Key2
 145     pmull    v\tmp1\().1q,v\aadhash\().1d,v\hashkey0_ext\().1d
 146     pmull2   v\tmp0\().1q,v\aadhash\().2d,v\hashkey0_ext\().2d
 147     eor      v\aadhash\().16b,v\aadhash\().16b,v\aadhash\().16b
 148
 149     .if  \left_count > 1
 150         ldr      q\hashkey0_ext,[\hashkey_adr],16
 151     .endif
 152     aes_encrypt_round   \enc_ctr,Key3
 153     eor      v\tmp0\().16b,v\tmp1\().16b,v\tmp0\().16b
 154
 155     aes_encrypt_round   \enc_ctr,Key4
 156     ext      v\tmp1\().16b,v\aadhash\().16b,v\tmp0\().16b,8 /*low */
 157     ext      v\tmp0\().16b,v\tmp0\().16b,v\aadhash\().16b,8 /* high */
 158     aes_encrypt_round   \enc_ctr,Key5
 159     eor      v\low\().16b,v\low\().16b,v\tmp1\().16b
 160     eor      v\high\().16b,v\high\().16b,v\tmp0\().16b
 161     aes_encrypt_round   \enc_ctr,Key6
 162     ldr      q\aadhash,[\dat_adr],16
 163     aes_encrypt_round   \enc_ctr,Key7
 164     aes_encrypt_round   \enc_ctr,Key8
 165     aes_encrypt_round   \enc_ctr,Key9
 166     aes_encrypt_round   \enc_ctr,Key10
 167     aes_encrypt_round   \enc_ctr,Key11
 168     aes_encrypt_round   \enc_ctr,Key12
 169     aese    v\enc_ctr\().16b,vKey13.16b
 170     eor     v\enc_ctr\().16b,v\enc_ctr\().16b,vKey14.16b
 171     eor     v\enc_ctr\().16b,v\enc_ctr\().16b,v\aadhash\().16b
 172     .ifc \is_enc , encrypt
 173         rbit    v\aadhash\().16b,v\enc_ctr\().16b
 174     .endif
 175     .ifc \is_enc , decrypt
 176         rbit    v\aadhash\().16b,v\aadhash\().16b
 177     .endif
 178     str     q\enc_ctr,[\out_adr],16
 179 .endm
 180
 181