ceph/src/spdk/intel-ipsec-mb/aes_keyexp_128.asm

   1 ;;
   2 ;; Copyright (c) 2012-2018, Intel Corporation
   3 ;;
   4 ;; Redistribution and use in source and binary forms, with or without
   5 ;; modification, are permitted provided that the following conditions are met:
   6 ;;
   7 ;;     * Redistributions of source code must retain the above copyright notice,
   8 ;;       this list of conditions and the following disclaimer.
   9 ;;     * Redistributions in binary form must reproduce the above copyright
  10 ;;       notice, this list of conditions and the following disclaimer in the
  11 ;;       documentation and/or other materials provided with the distribution.
  12 ;;     * Neither the name of Intel Corporation nor the names of its contributors
  13 ;;       may be used to endorse or promote products derived from this software
  14 ;;       without specific prior written permission.
  15 ;;
  16 ;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  17 ;; AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  18 ;; IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  19 ;; DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
  20 ;; FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  21 ;; DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
  22 ;; SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
  23 ;; CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  24 ;; OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  25 ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  26 ;;
  27
  28 ; Routine to do AES key expansion
  29 %include "os.asm"
  30
  31 %macro key_expansion_128_sse 0
  32         ;; Assumes the xmm3 includes all zeros at this point.
  33         pshufd  xmm2, xmm2, 11111111b
  34         shufps  xmm3, xmm1, 00010000b
  35         pxor    xmm1, xmm3
  36         shufps  xmm3, xmm1, 10001100b
  37         pxor    xmm1, xmm3
  38         pxor    xmm1, xmm2
  39 %endmacro
  40
  41 %macro key_expansion_128_avx 0
  42         ;; Assumes the xmm3 includes all zeros at this point.
  43         vpshufd xmm2, xmm2, 11111111b
  44         vshufps xmm3, xmm3, xmm1, 00010000b
  45         vpxor   xmm1, xmm1, xmm3
  46         vshufps xmm3, xmm3, xmm1, 10001100b
  47         vpxor   xmm1, xmm1, xmm3
  48         vpxor   xmm1, xmm1, xmm2
  49 %endmacro
  50
  51 %ifdef LINUX
  52 %define KEY             rdi
  53 %define EXP_ENC_KEYS    rsi
  54 %define EXP_DEC_KEYS    rdx
  55 %else
  56 %define KEY             rcx
  57 %define EXP_ENC_KEYS    rdx
  58 %define EXP_DEC_KEYS    r8
  59 %endif
  60
  61 section .text
  62
  63 ; void aes_keyexp_128(UINT128 *key,
  64 ;                     UINT128 *enc_exp_keys,
  65 ;                     UINT128 *dec_exp_keys);
  66 ;
  67 ; arg 1: rcx: pointer to key
  68 ; arg 2: rdx: pointer to expanded key array for encrypt
  69 ; arg 3: r8:  pointer to expanded key array for decrypt
  70 ;
  71 MKGLOBAL(aes_keyexp_128_sse,function,)
  72 aes_keyexp_128_sse:
  73         movdqu  xmm1, [KEY]     ; loading the AES key
  74         movdqa  [EXP_ENC_KEYS + 16*0], xmm1
  75         movdqa  [EXP_DEC_KEYS + 16*10], xmm1  ; Storing key in memory
  76         pxor    xmm3, xmm3
  77
  78         aeskeygenassist xmm2, xmm1, 0x1     ; Generating round key 1
  79         key_expansion_128_sse
  80         movdqa  [EXP_ENC_KEYS + 16*1], xmm1
  81         aesimc  xmm4, xmm1
  82         movdqa  [EXP_DEC_KEYS + 16*9], xmm4
  83
  84         aeskeygenassist xmm2, xmm1, 0x2     ; Generating round key 2
  85         key_expansion_128_sse
  86         movdqa  [EXP_ENC_KEYS + 16*2], xmm1
  87         aesimc  xmm5, xmm1
  88         movdqa  [EXP_DEC_KEYS + 16*8], xmm5
  89
  90         aeskeygenassist xmm2, xmm1, 0x4     ; Generating round key 3
  91         key_expansion_128_sse
  92         movdqa  [EXP_ENC_KEYS + 16*3], xmm1
  93         aesimc  xmm4, xmm1
  94         movdqa  [EXP_DEC_KEYS + 16*7], xmm4
  95
  96         aeskeygenassist xmm2, xmm1, 0x8     ; Generating round key 4
  97         key_expansion_128_sse
  98         movdqa  [EXP_ENC_KEYS + 16*4], xmm1
  99         aesimc  xmm5, xmm1
 100         movdqa  [EXP_DEC_KEYS + 16*6], xmm5
 101
 102         aeskeygenassist xmm2, xmm1, 0x10    ; Generating round key 5
 103         key_expansion_128_sse
 104         movdqa  [EXP_ENC_KEYS + 16*5], xmm1
 105         aesimc  xmm4, xmm1
 106         movdqa  [EXP_DEC_KEYS + 16*5], xmm4
 107
 108         aeskeygenassist xmm2, xmm1, 0x20    ; Generating round key 6
 109         key_expansion_128_sse
 110         movdqa  [EXP_ENC_KEYS + 16*6], xmm1
 111         aesimc  xmm5, xmm1
 112         movdqa  [EXP_DEC_KEYS + 16*4], xmm5
 113
 114         aeskeygenassist xmm2, xmm1, 0x40    ; Generating round key 7
 115         key_expansion_128_sse
 116         movdqa  [EXP_ENC_KEYS + 16*7], xmm1
 117         aesimc  xmm4, xmm1
 118         movdqa  [EXP_DEC_KEYS + 16*3], xmm4
 119
 120         aeskeygenassist xmm2, xmm1, 0x80    ; Generating round key 8
 121         key_expansion_128_sse
 122         movdqa  [EXP_ENC_KEYS + 16*8], xmm1
 123         aesimc  xmm5, xmm1
 124         movdqa  [EXP_DEC_KEYS + 16*2], xmm5
 125
 126         aeskeygenassist xmm2, xmm1, 0x1b    ; Generating round key 9
 127         key_expansion_128_sse
 128         movdqa  [EXP_ENC_KEYS + 16*9], xmm1
 129         aesimc  xmm4, xmm1
 130         movdqa  [EXP_DEC_KEYS + 16*1], xmm4
 131
 132         aeskeygenassist xmm2, xmm1, 0x36    ; Generating round key 10
 133         key_expansion_128_sse
 134         movdqa  [EXP_ENC_KEYS + 16*10], xmm1
 135         movdqa  [EXP_DEC_KEYS + 16*0], xmm1
 136
 137         ret
 138
 139 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 140 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 141 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 142 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 143
 144 MKGLOBAL(aes_keyexp_128_avx,function,)
 145 aes_keyexp_128_avx:
 146         vmovdqu xmm1, [KEY]     ; loading the AES key
 147         vmovdqa [EXP_ENC_KEYS + 16*0], xmm1
 148         vmovdqa [EXP_DEC_KEYS + 16*10], xmm1  ; Storing key in memory
 149         vpxor   xmm3, xmm3, xmm3
 150
 151         vaeskeygenassist        xmm2, xmm1, 0x1     ; Generating round key 1
 152         key_expansion_128_avx
 153         vmovdqa [EXP_ENC_KEYS + 16*1], xmm1
 154         vaesimc xmm4, xmm1
 155         vmovdqa [EXP_DEC_KEYS + 16*9], xmm4
 156
 157         vaeskeygenassist xmm2, xmm1, 0x2     ; Generating round key 2
 158         key_expansion_128_avx
 159         vmovdqa [EXP_ENC_KEYS + 16*2], xmm1
 160         vaesimc xmm5, xmm1
 161         vmovdqa [EXP_DEC_KEYS + 16*8], xmm5
 162
 163         vaeskeygenassist xmm2, xmm1, 0x4     ; Generating round key 3
 164         key_expansion_128_avx
 165         vmovdqa [EXP_ENC_KEYS + 16*3], xmm1
 166         vaesimc xmm4, xmm1
 167         vmovdqa [EXP_DEC_KEYS + 16*7], xmm4
 168
 169         vaeskeygenassist xmm2, xmm1, 0x8     ; Generating round key 4
 170         key_expansion_128_avx
 171         vmovdqa [EXP_ENC_KEYS + 16*4], xmm1
 172         vaesimc xmm5, xmm1
 173         vmovdqa [EXP_DEC_KEYS + 16*6], xmm5
 174
 175         vaeskeygenassist xmm2, xmm1, 0x10    ; Generating round key 5
 176         key_expansion_128_avx
 177         vmovdqa [EXP_ENC_KEYS + 16*5], xmm1
 178         vaesimc xmm4, xmm1
 179         vmovdqa [EXP_DEC_KEYS + 16*5], xmm4
 180
 181         vaeskeygenassist xmm2, xmm1, 0x20    ; Generating round key 6
 182         key_expansion_128_avx
 183         vmovdqa [EXP_ENC_KEYS + 16*6], xmm1
 184         vaesimc xmm5, xmm1
 185         vmovdqa [EXP_DEC_KEYS + 16*4], xmm5
 186
 187         vaeskeygenassist xmm2, xmm1, 0x40    ; Generating round key 7
 188         key_expansion_128_avx
 189         vmovdqa [EXP_ENC_KEYS + 16*7], xmm1
 190         vaesimc xmm4, xmm1
 191         vmovdqa [EXP_DEC_KEYS + 16*3], xmm4
 192
 193         vaeskeygenassist xmm2, xmm1, 0x80    ; Generating round key 8
 194         key_expansion_128_avx
 195         vmovdqa [EXP_ENC_KEYS + 16*8], xmm1
 196         vaesimc xmm5, xmm1
 197         vmovdqa [EXP_DEC_KEYS + 16*2], xmm5
 198
 199         vaeskeygenassist xmm2, xmm1, 0x1b    ; Generating round key 9
 200         key_expansion_128_avx
 201         vmovdqa [EXP_ENC_KEYS + 16*9], xmm1
 202         vaesimc xmm4, xmm1
 203         vmovdqa [EXP_DEC_KEYS + 16*1], xmm4
 204
 205         vaeskeygenassist xmm2, xmm1, 0x36    ; Generating round key 10
 206         key_expansion_128_avx
 207         vmovdqa [EXP_ENC_KEYS + 16*10], xmm1
 208         vmovdqa [EXP_DEC_KEYS + 16*0], xmm1
 209
 210         ret
 211
 212 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 213 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 214 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 215 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 216
 217 ; void aes_keyexp_128_enc_sse(UINT128 *key,
 218 ;                             UINT128 *enc_exp_keys);
 219 ;
 220 ; arg 1: rcx: pointer to key
 221 ; arg 2: rdx: pointer to expanded key array for encrypt
 222 ;
 223 MKGLOBAL(aes_keyexp_128_enc_sse,function,)
 224 aes_keyexp_128_enc_sse:
 225         movdqu  xmm1, [KEY]     ; loading the AES key
 226         movdqa  [EXP_ENC_KEYS + 16*0], xmm1
 227         pxor    xmm3, xmm3
 228
 229         aeskeygenassist xmm2, xmm1, 0x1     ; Generating round key 1
 230         key_expansion_128_sse
 231         movdqa  [EXP_ENC_KEYS + 16*1], xmm1
 232
 233         aeskeygenassist xmm2, xmm1, 0x2     ; Generating round key 2
 234         key_expansion_128_sse
 235         movdqa  [EXP_ENC_KEYS + 16*2], xmm1
 236
 237         aeskeygenassist xmm2, xmm1, 0x4     ; Generating round key 3
 238         key_expansion_128_sse
 239         movdqa  [EXP_ENC_KEYS + 16*3], xmm1
 240
 241         aeskeygenassist xmm2, xmm1, 0x8     ; Generating round key 4
 242         key_expansion_128_sse
 243         movdqa  [EXP_ENC_KEYS + 16*4], xmm1
 244
 245         aeskeygenassist xmm2, xmm1, 0x10    ; Generating round key 5
 246         key_expansion_128_sse
 247         movdqa  [EXP_ENC_KEYS + 16*5], xmm1
 248
 249         aeskeygenassist xmm2, xmm1, 0x20    ; Generating round key 6
 250         key_expansion_128_sse
 251         movdqa  [EXP_ENC_KEYS + 16*6], xmm1
 252
 253         aeskeygenassist xmm2, xmm1, 0x40    ; Generating round key 7
 254         key_expansion_128_sse
 255         movdqa  [EXP_ENC_KEYS + 16*7], xmm1
 256
 257         aeskeygenassist xmm2, xmm1, 0x80    ; Generating round key 8
 258         key_expansion_128_sse
 259         movdqa  [EXP_ENC_KEYS + 16*8], xmm1
 260
 261         aeskeygenassist xmm2, xmm1, 0x1b    ; Generating round key 9
 262         key_expansion_128_sse
 263         movdqa  [EXP_ENC_KEYS + 16*9], xmm1
 264
 265         aeskeygenassist xmm2, xmm1, 0x36    ; Generating round key 10
 266         key_expansion_128_sse
 267         movdqa  [EXP_ENC_KEYS + 16*10], xmm1
 268
 269         ret
 270
 271 MKGLOBAL(aes_keyexp_128_enc_avx,function,)
 272 aes_keyexp_128_enc_avx:
 273         vmovdqu xmm1, [KEY]     ; loading the AES key
 274         vmovdqa [EXP_ENC_KEYS + 16*0], xmm1
 275         vpxor   xmm3, xmm3, xmm3
 276
 277         vaeskeygenassist        xmm2, xmm1, 0x1     ; Generating round key 1
 278         key_expansion_128_avx
 279         vmovdqa [EXP_ENC_KEYS + 16*1], xmm1
 280
 281         vaeskeygenassist xmm2, xmm1, 0x2     ; Generating round key 2
 282         key_expansion_128_avx
 283         vmovdqa [EXP_ENC_KEYS + 16*2], xmm1
 284
 285         vaeskeygenassist xmm2, xmm1, 0x4     ; Generating round key 3
 286         key_expansion_128_avx
 287         vmovdqa [EXP_ENC_KEYS + 16*3], xmm1
 288
 289         vaeskeygenassist xmm2, xmm1, 0x8     ; Generating round key 4
 290         key_expansion_128_avx
 291         vmovdqa [EXP_ENC_KEYS + 16*4], xmm1
 292
 293         vaeskeygenassist xmm2, xmm1, 0x10    ; Generating round key 5
 294         key_expansion_128_avx
 295         vmovdqa [EXP_ENC_KEYS + 16*5], xmm1
 296
 297         vaeskeygenassist xmm2, xmm1, 0x20    ; Generating round key 6
 298         key_expansion_128_avx
 299         vmovdqa [EXP_ENC_KEYS + 16*6], xmm1
 300
 301         vaeskeygenassist xmm2, xmm1, 0x40    ; Generating round key 7
 302         key_expansion_128_avx
 303         vmovdqa [EXP_ENC_KEYS + 16*7], xmm1
 304
 305         vaeskeygenassist xmm2, xmm1, 0x80    ; Generating round key 8
 306         key_expansion_128_avx
 307         vmovdqa [EXP_ENC_KEYS + 16*8], xmm1
 308
 309         vaeskeygenassist xmm2, xmm1, 0x1b    ; Generating round key 9
 310         key_expansion_128_avx
 311         vmovdqa [EXP_ENC_KEYS + 16*9], xmm1
 312
 313         vaeskeygenassist xmm2, xmm1, 0x36    ; Generating round key 10
 314         key_expansion_128_avx
 315         vmovdqa [EXP_ENC_KEYS + 16*10], xmm1
 316
 317         ret
 318
 319 %ifdef LINUX
 320 section .note.GNU-stack noalloc noexec nowrite progbits
 321 %endif