1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /* Copyright (C) 2004-2006, Advanced Micro Devices, Inc.
5 #include <linux/module.h>
6 #include <linux/kernel.h>
8 #include <linux/pci_ids.h>
9 #include <linux/crypto.h>
10 #include <linux/spinlock.h>
11 #include <crypto/algapi.h>
12 #include <crypto/aes.h>
13 #include <crypto/internal/skcipher.h>
16 #include <linux/delay.h>
18 #include "geode-aes.h"
20 /* Static structures */
22 static void __iomem
*_iobase
;
23 static spinlock_t lock
;
25 /* Write a 128 bit field (either a writable key or IV) */
27 _writefield(u32 offset
, const void *value
)
31 for (i
= 0; i
< 4; i
++)
32 iowrite32(((const u32
*) value
)[i
], _iobase
+ offset
+ (i
* 4));
35 /* Read a 128 bit field (either a writable key or IV) */
37 _readfield(u32 offset
, void *value
)
41 for (i
= 0; i
< 4; i
++)
42 ((u32
*) value
)[i
] = ioread32(_iobase
+ offset
+ (i
* 4));
46 do_crypt(const void *src
, void *dst
, u32 len
, u32 flags
)
49 u32 counter
= AES_OP_TIMEOUT
;
51 iowrite32(virt_to_phys((void *)src
), _iobase
+ AES_SOURCEA_REG
);
52 iowrite32(virt_to_phys(dst
), _iobase
+ AES_DSTA_REG
);
53 iowrite32(len
, _iobase
+ AES_LENA_REG
);
55 /* Start the operation */
56 iowrite32(AES_CTRL_START
| flags
, _iobase
+ AES_CTRLA_REG
);
59 status
= ioread32(_iobase
+ AES_INTR_REG
);
61 } while (!(status
& AES_INTRA_PENDING
) && --counter
);
64 iowrite32((status
& 0xFF) | AES_INTRA_PENDING
, _iobase
+ AES_INTR_REG
);
65 return counter
? 0 : 1;
69 geode_aes_crypt(const struct geode_aes_tfm_ctx
*tctx
, const void *src
,
70 void *dst
, u32 len
, u8
*iv
, int mode
, int dir
)
76 /* If the source and destination is the same, then
77 * we need to turn on the coherent flags, otherwise
78 * we don't need to worry
81 flags
|= (AES_CTRL_DCA
| AES_CTRL_SCA
);
83 if (dir
== AES_DIR_ENCRYPT
)
84 flags
|= AES_CTRL_ENCRYPT
;
86 /* Start the critical section */
88 spin_lock_irqsave(&lock
, iflags
);
90 if (mode
== AES_MODE_CBC
) {
91 flags
|= AES_CTRL_CBC
;
92 _writefield(AES_WRITEIV0_REG
, iv
);
95 flags
|= AES_CTRL_WRKEY
;
96 _writefield(AES_WRITEKEY0_REG
, tctx
->key
);
98 ret
= do_crypt(src
, dst
, len
, flags
);
101 if (mode
== AES_MODE_CBC
)
102 _readfield(AES_WRITEIV0_REG
, iv
);
104 spin_unlock_irqrestore(&lock
, iflags
);
107 /* CRYPTO-API Functions */
109 static int geode_setkey_cip(struct crypto_tfm
*tfm
, const u8
*key
,
112 struct geode_aes_tfm_ctx
*tctx
= crypto_tfm_ctx(tfm
);
117 if (len
== AES_KEYSIZE_128
) {
118 memcpy(tctx
->key
, key
, len
);
122 if (len
!= AES_KEYSIZE_192
&& len
!= AES_KEYSIZE_256
) {
123 /* not supported at all */
124 tfm
->crt_flags
|= CRYPTO_TFM_RES_BAD_KEY_LEN
;
129 * The requested key size is not supported by HW, do a fallback
131 tctx
->fallback
.cip
->base
.crt_flags
&= ~CRYPTO_TFM_REQ_MASK
;
132 tctx
->fallback
.cip
->base
.crt_flags
|=
133 (tfm
->crt_flags
& CRYPTO_TFM_REQ_MASK
);
135 ret
= crypto_cipher_setkey(tctx
->fallback
.cip
, key
, len
);
137 tfm
->crt_flags
&= ~CRYPTO_TFM_RES_MASK
;
138 tfm
->crt_flags
|= (tctx
->fallback
.cip
->base
.crt_flags
&
139 CRYPTO_TFM_RES_MASK
);
144 static int geode_setkey_skcipher(struct crypto_skcipher
*tfm
, const u8
*key
,
147 struct geode_aes_tfm_ctx
*tctx
= crypto_skcipher_ctx(tfm
);
152 if (len
== AES_KEYSIZE_128
) {
153 memcpy(tctx
->key
, key
, len
);
157 if (len
!= AES_KEYSIZE_192
&& len
!= AES_KEYSIZE_256
) {
158 /* not supported at all */
159 crypto_skcipher_set_flags(tfm
, CRYPTO_TFM_RES_BAD_KEY_LEN
);
164 * The requested key size is not supported by HW, do a fallback
166 crypto_skcipher_clear_flags(tctx
->fallback
.skcipher
,
167 CRYPTO_TFM_REQ_MASK
);
168 crypto_skcipher_set_flags(tctx
->fallback
.skcipher
,
169 crypto_skcipher_get_flags(tfm
) &
170 CRYPTO_TFM_REQ_MASK
);
171 ret
= crypto_skcipher_setkey(tctx
->fallback
.skcipher
, key
, len
);
172 crypto_skcipher_set_flags(tfm
,
173 crypto_skcipher_get_flags(tctx
->fallback
.skcipher
) &
174 CRYPTO_TFM_RES_MASK
);
179 geode_encrypt(struct crypto_tfm
*tfm
, u8
*out
, const u8
*in
)
181 const struct geode_aes_tfm_ctx
*tctx
= crypto_tfm_ctx(tfm
);
183 if (unlikely(tctx
->keylen
!= AES_KEYSIZE_128
)) {
184 crypto_cipher_encrypt_one(tctx
->fallback
.cip
, out
, in
);
188 geode_aes_crypt(tctx
, in
, out
, AES_BLOCK_SIZE
, NULL
,
189 AES_MODE_ECB
, AES_DIR_ENCRYPT
);
194 geode_decrypt(struct crypto_tfm
*tfm
, u8
*out
, const u8
*in
)
196 const struct geode_aes_tfm_ctx
*tctx
= crypto_tfm_ctx(tfm
);
198 if (unlikely(tctx
->keylen
!= AES_KEYSIZE_128
)) {
199 crypto_cipher_decrypt_one(tctx
->fallback
.cip
, out
, in
);
203 geode_aes_crypt(tctx
, in
, out
, AES_BLOCK_SIZE
, NULL
,
204 AES_MODE_ECB
, AES_DIR_DECRYPT
);
207 static int fallback_init_cip(struct crypto_tfm
*tfm
)
209 const char *name
= crypto_tfm_alg_name(tfm
);
210 struct geode_aes_tfm_ctx
*tctx
= crypto_tfm_ctx(tfm
);
212 tctx
->fallback
.cip
= crypto_alloc_cipher(name
, 0,
213 CRYPTO_ALG_NEED_FALLBACK
);
215 if (IS_ERR(tctx
->fallback
.cip
)) {
216 printk(KERN_ERR
"Error allocating fallback algo %s\n", name
);
217 return PTR_ERR(tctx
->fallback
.cip
);
223 static void fallback_exit_cip(struct crypto_tfm
*tfm
)
225 struct geode_aes_tfm_ctx
*tctx
= crypto_tfm_ctx(tfm
);
227 crypto_free_cipher(tctx
->fallback
.cip
);
230 static struct crypto_alg geode_alg
= {
232 .cra_driver_name
= "geode-aes",
235 .cra_flags
= CRYPTO_ALG_TYPE_CIPHER
|
236 CRYPTO_ALG_NEED_FALLBACK
,
237 .cra_init
= fallback_init_cip
,
238 .cra_exit
= fallback_exit_cip
,
239 .cra_blocksize
= AES_BLOCK_SIZE
,
240 .cra_ctxsize
= sizeof(struct geode_aes_tfm_ctx
),
241 .cra_module
= THIS_MODULE
,
244 .cia_min_keysize
= AES_MIN_KEY_SIZE
,
245 .cia_max_keysize
= AES_MAX_KEY_SIZE
,
246 .cia_setkey
= geode_setkey_cip
,
247 .cia_encrypt
= geode_encrypt
,
248 .cia_decrypt
= geode_decrypt
253 static int geode_init_skcipher(struct crypto_skcipher
*tfm
)
255 const char *name
= crypto_tfm_alg_name(&tfm
->base
);
256 struct geode_aes_tfm_ctx
*tctx
= crypto_skcipher_ctx(tfm
);
258 tctx
->fallback
.skcipher
=
259 crypto_alloc_skcipher(name
, 0, CRYPTO_ALG_NEED_FALLBACK
|
261 if (IS_ERR(tctx
->fallback
.skcipher
)) {
262 printk(KERN_ERR
"Error allocating fallback algo %s\n", name
);
263 return PTR_ERR(tctx
->fallback
.skcipher
);
266 crypto_skcipher_set_reqsize(tfm
, sizeof(struct skcipher_request
) +
267 crypto_skcipher_reqsize(tctx
->fallback
.skcipher
));
271 static void geode_exit_skcipher(struct crypto_skcipher
*tfm
)
273 struct geode_aes_tfm_ctx
*tctx
= crypto_skcipher_ctx(tfm
);
275 crypto_free_skcipher(tctx
->fallback
.skcipher
);
278 static int geode_skcipher_crypt(struct skcipher_request
*req
, int mode
, int dir
)
280 struct crypto_skcipher
*tfm
= crypto_skcipher_reqtfm(req
);
281 const struct geode_aes_tfm_ctx
*tctx
= crypto_skcipher_ctx(tfm
);
282 struct skcipher_walk walk
;
286 if (unlikely(tctx
->keylen
!= AES_KEYSIZE_128
)) {
287 struct skcipher_request
*subreq
= skcipher_request_ctx(req
);
290 skcipher_request_set_tfm(subreq
, tctx
->fallback
.skcipher
);
291 if (dir
== AES_DIR_DECRYPT
)
292 return crypto_skcipher_decrypt(subreq
);
294 return crypto_skcipher_encrypt(subreq
);
297 err
= skcipher_walk_virt(&walk
, req
, false);
299 while ((nbytes
= walk
.nbytes
) != 0) {
300 geode_aes_crypt(tctx
, walk
.src
.virt
.addr
, walk
.dst
.virt
.addr
,
301 round_down(nbytes
, AES_BLOCK_SIZE
),
303 err
= skcipher_walk_done(&walk
, nbytes
% AES_BLOCK_SIZE
);
309 static int geode_cbc_encrypt(struct skcipher_request
*req
)
311 return geode_skcipher_crypt(req
, AES_MODE_CBC
, AES_DIR_ENCRYPT
);
314 static int geode_cbc_decrypt(struct skcipher_request
*req
)
316 return geode_skcipher_crypt(req
, AES_MODE_CBC
, AES_DIR_DECRYPT
);
319 static int geode_ecb_encrypt(struct skcipher_request
*req
)
321 return geode_skcipher_crypt(req
, AES_MODE_ECB
, AES_DIR_ENCRYPT
);
324 static int geode_ecb_decrypt(struct skcipher_request
*req
)
326 return geode_skcipher_crypt(req
, AES_MODE_ECB
, AES_DIR_DECRYPT
);
329 static struct skcipher_alg geode_skcipher_algs
[] = {
331 .base
.cra_name
= "cbc(aes)",
332 .base
.cra_driver_name
= "cbc-aes-geode",
333 .base
.cra_priority
= 400,
334 .base
.cra_flags
= CRYPTO_ALG_KERN_DRIVER_ONLY
|
335 CRYPTO_ALG_NEED_FALLBACK
,
336 .base
.cra_blocksize
= AES_BLOCK_SIZE
,
337 .base
.cra_ctxsize
= sizeof(struct geode_aes_tfm_ctx
),
338 .base
.cra_alignmask
= 15,
339 .base
.cra_module
= THIS_MODULE
,
340 .init
= geode_init_skcipher
,
341 .exit
= geode_exit_skcipher
,
342 .setkey
= geode_setkey_skcipher
,
343 .encrypt
= geode_cbc_encrypt
,
344 .decrypt
= geode_cbc_decrypt
,
345 .min_keysize
= AES_MIN_KEY_SIZE
,
346 .max_keysize
= AES_MAX_KEY_SIZE
,
347 .ivsize
= AES_BLOCK_SIZE
,
349 .base
.cra_name
= "ecb(aes)",
350 .base
.cra_driver_name
= "ecb-aes-geode",
351 .base
.cra_priority
= 400,
352 .base
.cra_flags
= CRYPTO_ALG_KERN_DRIVER_ONLY
|
353 CRYPTO_ALG_NEED_FALLBACK
,
354 .base
.cra_blocksize
= AES_BLOCK_SIZE
,
355 .base
.cra_ctxsize
= sizeof(struct geode_aes_tfm_ctx
),
356 .base
.cra_alignmask
= 15,
357 .base
.cra_module
= THIS_MODULE
,
358 .init
= geode_init_skcipher
,
359 .exit
= geode_exit_skcipher
,
360 .setkey
= geode_setkey_skcipher
,
361 .encrypt
= geode_ecb_encrypt
,
362 .decrypt
= geode_ecb_decrypt
,
363 .min_keysize
= AES_MIN_KEY_SIZE
,
364 .max_keysize
= AES_MAX_KEY_SIZE
,
368 static void geode_aes_remove(struct pci_dev
*dev
)
370 crypto_unregister_alg(&geode_alg
);
371 crypto_unregister_skciphers(geode_skcipher_algs
,
372 ARRAY_SIZE(geode_skcipher_algs
));
374 pci_iounmap(dev
, _iobase
);
377 pci_release_regions(dev
);
378 pci_disable_device(dev
);
382 static int geode_aes_probe(struct pci_dev
*dev
, const struct pci_device_id
*id
)
386 ret
= pci_enable_device(dev
);
390 ret
= pci_request_regions(dev
, "geode-aes");
394 _iobase
= pci_iomap(dev
, 0, 0);
396 if (_iobase
== NULL
) {
401 spin_lock_init(&lock
);
403 /* Clear any pending activity */
404 iowrite32(AES_INTR_PENDING
| AES_INTR_MASK
, _iobase
+ AES_INTR_REG
);
406 ret
= crypto_register_alg(&geode_alg
);
410 ret
= crypto_register_skciphers(geode_skcipher_algs
,
411 ARRAY_SIZE(geode_skcipher_algs
));
415 dev_notice(&dev
->dev
, "GEODE AES engine enabled.\n");
419 crypto_unregister_alg(&geode_alg
);
422 pci_iounmap(dev
, _iobase
);
425 pci_release_regions(dev
);
428 pci_disable_device(dev
);
430 dev_err(&dev
->dev
, "GEODE AES initialization failed.\n");
434 static struct pci_device_id geode_aes_tbl
[] = {
435 { PCI_VDEVICE(AMD
, PCI_DEVICE_ID_AMD_LX_AES
), },
439 MODULE_DEVICE_TABLE(pci
, geode_aes_tbl
);
441 static struct pci_driver geode_aes_driver
= {
442 .name
= "Geode LX AES",
443 .id_table
= geode_aes_tbl
,
444 .probe
= geode_aes_probe
,
445 .remove
= geode_aes_remove
,
448 module_pci_driver(geode_aes_driver
);
450 MODULE_AUTHOR("Advanced Micro Devices, Inc.");
451 MODULE_DESCRIPTION("Geode LX Hardware AES driver");
452 MODULE_LICENSE("GPL");