arch/arm64/crypto/aes-ce.S

   1 /*
   2  * linux/arch/arm64/crypto/aes-ce.S - AES cipher for ARMv8 with
   3  *                                    Crypto Extensions
   4  *
   5  * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License version 2 as
   9  * published by the Free Software Foundation.
  10  */
  11
  12 #include <linux/linkage.h>
  13 #include <asm/assembler.h>
  14
  15 #define AES_ENTRY(func)         ENTRY(ce_ ## func)
  16 #define AES_ENDPROC(func)       ENDPROC(ce_ ## func)
  17
  18         .arch           armv8-a+crypto
  19
  20         xtsmask         .req    v16
  21
  22         .macro          xts_reload_mask, tmp
  23         .endm
  24
  25         /* preload all round keys */
  26         .macro          load_round_keys, rounds, rk
  27         cmp             \rounds, #12
  28         blo             2222f           /* 128 bits */
  29         beq             1111f           /* 192 bits */
  30         ld1             {v17.4s-v18.4s}, [\rk], #32
  31 1111:   ld1             {v19.4s-v20.4s}, [\rk], #32
  32 2222:   ld1             {v21.4s-v24.4s}, [\rk], #64
  33         ld1             {v25.4s-v28.4s}, [\rk], #64
  34         ld1             {v29.4s-v31.4s}, [\rk]
  35         .endm
  36
  37         /* prepare for encryption with key in rk[] */
  38         .macro          enc_prepare, rounds, rk, temp
  39         mov             \temp, \rk
  40         load_round_keys \rounds, \temp
  41         .endm
  42
  43         /* prepare for encryption (again) but with new key in rk[] */
  44         .macro          enc_switch_key, rounds, rk, temp
  45         mov             \temp, \rk
  46         load_round_keys \rounds, \temp
  47         .endm
  48
  49         /* prepare for decryption with key in rk[] */
  50         .macro          dec_prepare, rounds, rk, temp
  51         mov             \temp, \rk
  52         load_round_keys \rounds, \temp
  53         .endm
  54
  55         .macro          do_enc_Nx, de, mc, k, i0, i1, i2, i3
  56         aes\de          \i0\().16b, \k\().16b
  57         aes\mc          \i0\().16b, \i0\().16b
  58         .ifnb           \i1
  59         aes\de          \i1\().16b, \k\().16b
  60         aes\mc          \i1\().16b, \i1\().16b
  61         .ifnb           \i3
  62         aes\de          \i2\().16b, \k\().16b
  63         aes\mc          \i2\().16b, \i2\().16b
  64         aes\de          \i3\().16b, \k\().16b
  65         aes\mc          \i3\().16b, \i3\().16b
  66         .endif
  67         .endif
  68         .endm
  69
  70         /* up to 4 interleaved encryption rounds with the same round key */
  71         .macro          round_Nx, enc, k, i0, i1, i2, i3
  72         .ifc            \enc, e
  73         do_enc_Nx       e, mc, \k, \i0, \i1, \i2, \i3
  74         .else
  75         do_enc_Nx       d, imc, \k, \i0, \i1, \i2, \i3
  76         .endif
  77         .endm
  78
  79         /* up to 4 interleaved final rounds */
  80         .macro          fin_round_Nx, de, k, k2, i0, i1, i2, i3
  81         aes\de          \i0\().16b, \k\().16b
  82         .ifnb           \i1
  83         aes\de          \i1\().16b, \k\().16b
  84         .ifnb           \i3
  85         aes\de          \i2\().16b, \k\().16b
  86         aes\de          \i3\().16b, \k\().16b
  87         .endif
  88         .endif
  89         eor             \i0\().16b, \i0\().16b, \k2\().16b
  90         .ifnb           \i1
  91         eor             \i1\().16b, \i1\().16b, \k2\().16b
  92         .ifnb           \i3
  93         eor             \i2\().16b, \i2\().16b, \k2\().16b
  94         eor             \i3\().16b, \i3\().16b, \k2\().16b
  95         .endif
  96         .endif
  97         .endm
  98
  99         /* up to 4 interleaved blocks */
 100         .macro          do_block_Nx, enc, rounds, i0, i1, i2, i3
 101         cmp             \rounds, #12
 102         blo             2222f           /* 128 bits */
 103         beq             1111f           /* 192 bits */
 104         round_Nx        \enc, v17, \i0, \i1, \i2, \i3
 105         round_Nx        \enc, v18, \i0, \i1, \i2, \i3
 106 1111:   round_Nx        \enc, v19, \i0, \i1, \i2, \i3
 107         round_Nx        \enc, v20, \i0, \i1, \i2, \i3
 108 2222:   .irp            key, v21, v22, v23, v24, v25, v26, v27, v28, v29
 109         round_Nx        \enc, \key, \i0, \i1, \i2, \i3
 110         .endr
 111         fin_round_Nx    \enc, v30, v31, \i0, \i1, \i2, \i3
 112         .endm
 113
 114         .macro          encrypt_block, in, rounds, t0, t1, t2
 115         do_block_Nx     e, \rounds, \in
 116         .endm
 117
 118         .macro          encrypt_block2x, i0, i1, rounds, t0, t1, t2
 119         do_block_Nx     e, \rounds, \i0, \i1
 120         .endm
 121
 122         .macro          encrypt_block4x, i0, i1, i2, i3, rounds, t0, t1, t2
 123         do_block_Nx     e, \rounds, \i0, \i1, \i2, \i3
 124         .endm
 125
 126         .macro          decrypt_block, in, rounds, t0, t1, t2
 127         do_block_Nx     d, \rounds, \in
 128         .endm
 129
 130         .macro          decrypt_block2x, i0, i1, rounds, t0, t1, t2
 131         do_block_Nx     d, \rounds, \i0, \i1
 132         .endm
 133
 134         .macro          decrypt_block4x, i0, i1, i2, i3, rounds, t0, t1, t2
 135         do_block_Nx     d, \rounds, \i0, \i1, \i2, \i3
 136         .endm
 137
 138 #include "aes-modes.S"