2 * linux/arch/arm64/crypto/aes-ce.S - AES cipher for ARMv8 with
5 * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org>
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
12 #include <linux/linkage.h>
13 #include <asm/assembler.h>
15 #define AES_ENTRY(func) ENTRY(ce_ ## func)
16 #define AES_ENDPROC(func) ENDPROC(ce_ ## func)
22 .macro xts_reload_mask, tmp
25 /* preload all round keys */
26 .macro load_round_keys, rounds, rk
28 blo 2222f /* 128 bits */
29 beq 1111f /* 192 bits */
30 ld1 {v17.4s-v18.4s}, [\rk], #32
31 1111: ld1 {v19.4s-v20.4s}, [\rk], #32
32 2222: ld1 {v21.4s-v24.4s}, [\rk], #64
33 ld1 {v25.4s-v28.4s}, [\rk], #64
34 ld1 {v29.4s-v31.4s}, [\rk]
37 /* prepare for encryption with key in rk[] */
38 .macro enc_prepare, rounds, rk, temp
40 load_round_keys \rounds, \temp
43 /* prepare for encryption (again) but with new key in rk[] */
44 .macro enc_switch_key, rounds, rk, temp
46 load_round_keys \rounds, \temp
49 /* prepare for decryption with key in rk[] */
50 .macro dec_prepare, rounds, rk, temp
52 load_round_keys \rounds, \temp
55 .macro do_enc_Nx, de, mc, k, i0, i1, i2, i3
56 aes\de \i0\().16b, \k\().16b
57 aes\mc \i0\().16b, \i0\().16b
59 aes\de \i1\().16b, \k\().16b
60 aes\mc \i1\().16b, \i1\().16b
62 aes\de \i2\().16b, \k\().16b
63 aes\mc \i2\().16b, \i2\().16b
64 aes\de \i3\().16b, \k\().16b
65 aes\mc \i3\().16b, \i3\().16b
70 /* up to 4 interleaved encryption rounds with the same round key */
71 .macro round_Nx, enc, k, i0, i1, i2, i3
73 do_enc_Nx e, mc, \k, \i0, \i1, \i2, \i3
75 do_enc_Nx d, imc, \k, \i0, \i1, \i2, \i3
79 /* up to 4 interleaved final rounds */
80 .macro fin_round_Nx, de, k, k2, i0, i1, i2, i3
81 aes\de \i0\().16b, \k\().16b
83 aes\de \i1\().16b, \k\().16b
85 aes\de \i2\().16b, \k\().16b
86 aes\de \i3\().16b, \k\().16b
89 eor \i0\().16b, \i0\().16b, \k2\().16b
91 eor \i1\().16b, \i1\().16b, \k2\().16b
93 eor \i2\().16b, \i2\().16b, \k2\().16b
94 eor \i3\().16b, \i3\().16b, \k2\().16b
99 /* up to 4 interleaved blocks */
100 .macro do_block_Nx, enc, rounds, i0, i1, i2, i3
102 blo 2222f /* 128 bits */
103 beq 1111f /* 192 bits */
104 round_Nx \enc, v17, \i0, \i1, \i2, \i3
105 round_Nx \enc, v18, \i0, \i1, \i2, \i3
106 1111: round_Nx \enc, v19, \i0, \i1, \i2, \i3
107 round_Nx \enc, v20, \i0, \i1, \i2, \i3
108 2222: .irp key, v21, v22, v23, v24, v25, v26, v27, v28, v29
109 round_Nx \enc, \key, \i0, \i1, \i2, \i3
111 fin_round_Nx \enc, v30, v31, \i0, \i1, \i2, \i3
114 .macro encrypt_block, in, rounds, t0, t1, t2
115 do_block_Nx e, \rounds, \in
118 .macro encrypt_block2x, i0, i1, rounds, t0, t1, t2
119 do_block_Nx e, \rounds, \i0, \i1
122 .macro encrypt_block4x, i0, i1, i2, i3, rounds, t0, t1, t2
123 do_block_Nx e, \rounds, \i0, \i1, \i2, \i3
126 .macro decrypt_block, in, rounds, t0, t1, t2
127 do_block_Nx d, \rounds, \in
130 .macro decrypt_block2x, i0, i1, rounds, t0, t1, t2
131 do_block_Nx d, \rounds, \i0, \i1
134 .macro decrypt_block4x, i0, i1, i2, i3, rounds, t0, t1, t2
135 do_block_Nx d, \rounds, \i0, \i1, \i2, \i3
138 #include "aes-modes.S"