drivers/crypto/geode-aes.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2  /* Copyright (C) 2004-2006, Advanced Micro Devices, Inc.
   3   */
   4
   5 #include <linux/module.h>
   6 #include <linux/kernel.h>
   7 #include <linux/pci.h>
   8 #include <linux/pci_ids.h>
   9 #include <linux/crypto.h>
  10 #include <linux/spinlock.h>
  11 #include <crypto/algapi.h>
  12 #include <crypto/aes.h>
  13
  14 #include <linux/io.h>
  15 #include <linux/delay.h>
  16
  17 #include "geode-aes.h"
  18
  19 /* Static structures */
  20
  21 static void __iomem *_iobase;
  22 static spinlock_t lock;
  23
  24 /* Write a 128 bit field (either a writable key or IV) */
  25 static inline void
  26 _writefield(u32 offset, void *value)
  27 {
  28         int i;
  29
  30         for (i = 0; i < 4; i++)
  31                 iowrite32(((u32 *) value)[i], _iobase + offset + (i * 4));
  32 }
  33
  34 /* Read a 128 bit field (either a writable key or IV) */
  35 static inline void
  36 _readfield(u32 offset, void *value)
  37 {
  38         int i;
  39
  40         for (i = 0; i < 4; i++)
  41                 ((u32 *) value)[i] = ioread32(_iobase + offset + (i * 4));
  42 }
  43
  44 static int
  45 do_crypt(void *src, void *dst, int len, u32 flags)
  46 {
  47         u32 status;
  48         u32 counter = AES_OP_TIMEOUT;
  49
  50         iowrite32(virt_to_phys(src), _iobase + AES_SOURCEA_REG);
  51         iowrite32(virt_to_phys(dst), _iobase + AES_DSTA_REG);
  52         iowrite32(len,  _iobase + AES_LENA_REG);
  53
  54         /* Start the operation */
  55         iowrite32(AES_CTRL_START | flags, _iobase + AES_CTRLA_REG);
  56
  57         do {
  58                 status = ioread32(_iobase + AES_INTR_REG);
  59                 cpu_relax();
  60         } while (!(status & AES_INTRA_PENDING) && --counter);
  61
  62         /* Clear the event */
  63         iowrite32((status & 0xFF) | AES_INTRA_PENDING, _iobase + AES_INTR_REG);
  64         return counter ? 0 : 1;
  65 }
  66
  67 static unsigned int
  68 geode_aes_crypt(struct geode_aes_op *op)
  69 {
  70         u32 flags = 0;
  71         unsigned long iflags;
  72         int ret;
  73
  74         if (op->len == 0)
  75                 return 0;
  76
  77         /* If the source and destination is the same, then
  78          * we need to turn on the coherent flags, otherwise
  79          * we don't need to worry
  80          */
  81
  82         flags |= (AES_CTRL_DCA | AES_CTRL_SCA);
  83
  84         if (op->dir == AES_DIR_ENCRYPT)
  85                 flags |= AES_CTRL_ENCRYPT;
  86
  87         /* Start the critical section */
  88
  89         spin_lock_irqsave(&lock, iflags);
  90
  91         if (op->mode == AES_MODE_CBC) {
  92                 flags |= AES_CTRL_CBC;
  93                 _writefield(AES_WRITEIV0_REG, op->iv);
  94         }
  95
  96         if (!(op->flags & AES_FLAGS_HIDDENKEY)) {
  97                 flags |= AES_CTRL_WRKEY;
  98                 _writefield(AES_WRITEKEY0_REG, op->key);
  99         }
 100
 101         ret = do_crypt(op->src, op->dst, op->len, flags);
 102         BUG_ON(ret);
 103
 104         if (op->mode == AES_MODE_CBC)
 105                 _readfield(AES_WRITEIV0_REG, op->iv);
 106
 107         spin_unlock_irqrestore(&lock, iflags);
 108
 109         return op->len;
 110 }
 111
 112 /* CRYPTO-API Functions */
 113
 114 static int geode_setkey_cip(struct crypto_tfm *tfm, const u8 *key,
 115                 unsigned int len)
 116 {
 117         struct geode_aes_op *op = crypto_tfm_ctx(tfm);
 118         unsigned int ret;
 119
 120         op->keylen = len;
 121
 122         if (len == AES_KEYSIZE_128) {
 123                 memcpy(op->key, key, len);
 124                 return 0;
 125         }
 126
 127         if (len != AES_KEYSIZE_192 && len != AES_KEYSIZE_256) {
 128                 /* not supported at all */
 129                 tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
 130                 return -EINVAL;
 131         }
 132
 133         /*
 134          * The requested key size is not supported by HW, do a fallback
 135          */
 136         op->fallback.cip->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
 137         op->fallback.cip->base.crt_flags |= (tfm->crt_flags & CRYPTO_TFM_REQ_MASK);
 138
 139         ret = crypto_cipher_setkey(op->fallback.cip, key, len);
 140         if (ret) {
 141                 tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK;
 142                 tfm->crt_flags |= (op->fallback.cip->base.crt_flags & CRYPTO_TFM_RES_MASK);
 143         }
 144         return ret;
 145 }
 146
 147 static int geode_setkey_blk(struct crypto_tfm *tfm, const u8 *key,
 148                 unsigned int len)
 149 {
 150         struct geode_aes_op *op = crypto_tfm_ctx(tfm);
 151         unsigned int ret;
 152
 153         op->keylen = len;
 154
 155         if (len == AES_KEYSIZE_128) {
 156                 memcpy(op->key, key, len);
 157                 return 0;
 158         }
 159
 160         if (len != AES_KEYSIZE_192 && len != AES_KEYSIZE_256) {
 161                 /* not supported at all */
 162                 tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
 163                 return -EINVAL;
 164         }
 165
 166         /*
 167          * The requested key size is not supported by HW, do a fallback
 168          */
 169         op->fallback.blk->base.crt_flags &= ~CRYPTO_TFM_REQ_MASK;
 170         op->fallback.blk->base.crt_flags |= (tfm->crt_flags & CRYPTO_TFM_REQ_MASK);
 171
 172         ret = crypto_blkcipher_setkey(op->fallback.blk, key, len);
 173         if (ret) {
 174                 tfm->crt_flags &= ~CRYPTO_TFM_RES_MASK;
 175                 tfm->crt_flags |= (op->fallback.blk->base.crt_flags & CRYPTO_TFM_RES_MASK);
 176         }
 177         return ret;
 178 }
 179
 180 static int fallback_blk_dec(struct blkcipher_desc *desc,
 181                 struct scatterlist *dst, struct scatterlist *src,
 182                 unsigned int nbytes)
 183 {
 184         unsigned int ret;
 185         struct crypto_blkcipher *tfm;
 186         struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm);
 187
 188         tfm = desc->tfm;
 189         desc->tfm = op->fallback.blk;
 190
 191         ret = crypto_blkcipher_decrypt_iv(desc, dst, src, nbytes);
 192
 193         desc->tfm = tfm;
 194         return ret;
 195 }
 196 static int fallback_blk_enc(struct blkcipher_desc *desc,
 197                 struct scatterlist *dst, struct scatterlist *src,
 198                 unsigned int nbytes)
 199 {
 200         unsigned int ret;
 201         struct crypto_blkcipher *tfm;
 202         struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm);
 203
 204         tfm = desc->tfm;
 205         desc->tfm = op->fallback.blk;
 206
 207         ret = crypto_blkcipher_encrypt_iv(desc, dst, src, nbytes);
 208
 209         desc->tfm = tfm;
 210         return ret;
 211 }
 212
 213 static void
 214 geode_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
 215 {
 216         struct geode_aes_op *op = crypto_tfm_ctx(tfm);
 217
 218         if (unlikely(op->keylen != AES_KEYSIZE_128)) {
 219                 crypto_cipher_encrypt_one(op->fallback.cip, out, in);
 220                 return;
 221         }
 222
 223         op->src = (void *) in;
 224         op->dst = (void *) out;
 225         op->mode = AES_MODE_ECB;
 226         op->flags = 0;
 227         op->len = AES_BLOCK_SIZE;
 228         op->dir = AES_DIR_ENCRYPT;
 229
 230         geode_aes_crypt(op);
 231 }
 232
 233
 234 static void
 235 geode_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
 236 {
 237         struct geode_aes_op *op = crypto_tfm_ctx(tfm);
 238
 239         if (unlikely(op->keylen != AES_KEYSIZE_128)) {
 240                 crypto_cipher_decrypt_one(op->fallback.cip, out, in);
 241                 return;
 242         }
 243
 244         op->src = (void *) in;
 245         op->dst = (void *) out;
 246         op->mode = AES_MODE_ECB;
 247         op->flags = 0;
 248         op->len = AES_BLOCK_SIZE;
 249         op->dir = AES_DIR_DECRYPT;
 250
 251         geode_aes_crypt(op);
 252 }
 253
 254 static int fallback_init_cip(struct crypto_tfm *tfm)
 255 {
 256         const char *name = crypto_tfm_alg_name(tfm);
 257         struct geode_aes_op *op = crypto_tfm_ctx(tfm);
 258
 259         op->fallback.cip = crypto_alloc_cipher(name, 0,
 260                                                CRYPTO_ALG_NEED_FALLBACK);
 261
 262         if (IS_ERR(op->fallback.cip)) {
 263                 printk(KERN_ERR "Error allocating fallback algo %s\n", name);
 264                 return PTR_ERR(op->fallback.cip);
 265         }
 266
 267         return 0;
 268 }
 269
 270 static void fallback_exit_cip(struct crypto_tfm *tfm)
 271 {
 272         struct geode_aes_op *op = crypto_tfm_ctx(tfm);
 273
 274         crypto_free_cipher(op->fallback.cip);
 275         op->fallback.cip = NULL;
 276 }
 277
 278 static struct crypto_alg geode_alg = {
 279         .cra_name                       =       "aes",
 280         .cra_driver_name        =       "geode-aes",
 281         .cra_priority           =       300,
 282         .cra_alignmask          =       15,
 283         .cra_flags                      =       CRYPTO_ALG_TYPE_CIPHER |
 284                                                         CRYPTO_ALG_NEED_FALLBACK,
 285         .cra_init                       =       fallback_init_cip,
 286         .cra_exit                       =       fallback_exit_cip,
 287         .cra_blocksize          =       AES_BLOCK_SIZE,
 288         .cra_ctxsize            =       sizeof(struct geode_aes_op),
 289         .cra_module                     =       THIS_MODULE,
 290         .cra_u                          =       {
 291                 .cipher =       {
 292                         .cia_min_keysize        =       AES_MIN_KEY_SIZE,
 293                         .cia_max_keysize        =       AES_MAX_KEY_SIZE,
 294                         .cia_setkey                     =       geode_setkey_cip,
 295                         .cia_encrypt            =       geode_encrypt,
 296                         .cia_decrypt            =       geode_decrypt
 297                 }
 298         }
 299 };
 300
 301 static int
 302 geode_cbc_decrypt(struct blkcipher_desc *desc,
 303                   struct scatterlist *dst, struct scatterlist *src,
 304                   unsigned int nbytes)
 305 {
 306         struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm);
 307         struct blkcipher_walk walk;
 308         int err, ret;
 309
 310         if (unlikely(op->keylen != AES_KEYSIZE_128))
 311                 return fallback_blk_dec(desc, dst, src, nbytes);
 312
 313         blkcipher_walk_init(&walk, dst, src, nbytes);
 314         err = blkcipher_walk_virt(desc, &walk);
 315         op->iv = walk.iv;
 316
 317         while ((nbytes = walk.nbytes)) {
 318                 op->src = walk.src.virt.addr,
 319                 op->dst = walk.dst.virt.addr;
 320                 op->mode = AES_MODE_CBC;
 321                 op->len = nbytes - (nbytes % AES_BLOCK_SIZE);
 322                 op->dir = AES_DIR_DECRYPT;
 323
 324                 ret = geode_aes_crypt(op);
 325
 326                 nbytes -= ret;
 327                 err = blkcipher_walk_done(desc, &walk, nbytes);
 328         }
 329
 330         return err;
 331 }
 332
 333 static int
 334 geode_cbc_encrypt(struct blkcipher_desc *desc,
 335                   struct scatterlist *dst, struct scatterlist *src,
 336                   unsigned int nbytes)
 337 {
 338         struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm);
 339         struct blkcipher_walk walk;
 340         int err, ret;
 341
 342         if (unlikely(op->keylen != AES_KEYSIZE_128))
 343                 return fallback_blk_enc(desc, dst, src, nbytes);
 344
 345         blkcipher_walk_init(&walk, dst, src, nbytes);
 346         err = blkcipher_walk_virt(desc, &walk);
 347         op->iv = walk.iv;
 348
 349         while ((nbytes = walk.nbytes)) {
 350                 op->src = walk.src.virt.addr,
 351                 op->dst = walk.dst.virt.addr;
 352                 op->mode = AES_MODE_CBC;
 353                 op->len = nbytes - (nbytes % AES_BLOCK_SIZE);
 354                 op->dir = AES_DIR_ENCRYPT;
 355
 356                 ret = geode_aes_crypt(op);
 357                 nbytes -= ret;
 358                 err = blkcipher_walk_done(desc, &walk, nbytes);
 359         }
 360
 361         return err;
 362 }
 363
 364 static int fallback_init_blk(struct crypto_tfm *tfm)
 365 {
 366         const char *name = crypto_tfm_alg_name(tfm);
 367         struct geode_aes_op *op = crypto_tfm_ctx(tfm);
 368
 369         op->fallback.blk = crypto_alloc_blkcipher(name, 0,
 370                         CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK);
 371
 372         if (IS_ERR(op->fallback.blk)) {
 373                 printk(KERN_ERR "Error allocating fallback algo %s\n", name);
 374                 return PTR_ERR(op->fallback.blk);
 375         }
 376
 377         return 0;
 378 }
 379
 380 static void fallback_exit_blk(struct crypto_tfm *tfm)
 381 {
 382         struct geode_aes_op *op = crypto_tfm_ctx(tfm);
 383
 384         crypto_free_blkcipher(op->fallback.blk);
 385         op->fallback.blk = NULL;
 386 }
 387
 388 static struct crypto_alg geode_cbc_alg = {
 389         .cra_name               =       "cbc(aes)",
 390         .cra_driver_name        =       "cbc-aes-geode",
 391         .cra_priority           =       400,
 392         .cra_flags                      =       CRYPTO_ALG_TYPE_BLKCIPHER |
 393                                                 CRYPTO_ALG_KERN_DRIVER_ONLY |
 394                                                 CRYPTO_ALG_NEED_FALLBACK,
 395         .cra_init                       =       fallback_init_blk,
 396         .cra_exit                       =       fallback_exit_blk,
 397         .cra_blocksize          =       AES_BLOCK_SIZE,
 398         .cra_ctxsize            =       sizeof(struct geode_aes_op),
 399         .cra_alignmask          =       15,
 400         .cra_type                       =       &crypto_blkcipher_type,
 401         .cra_module                     =       THIS_MODULE,
 402         .cra_u                          =       {
 403                 .blkcipher      =       {
 404                         .min_keysize    =       AES_MIN_KEY_SIZE,
 405                         .max_keysize    =       AES_MAX_KEY_SIZE,
 406                         .setkey                 =       geode_setkey_blk,
 407                         .encrypt                =       geode_cbc_encrypt,
 408                         .decrypt                =       geode_cbc_decrypt,
 409                         .ivsize                 =       AES_BLOCK_SIZE,
 410                 }
 411         }
 412 };
 413
 414 static int
 415 geode_ecb_decrypt(struct blkcipher_desc *desc,
 416                   struct scatterlist *dst, struct scatterlist *src,
 417                   unsigned int nbytes)
 418 {
 419         struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm);
 420         struct blkcipher_walk walk;
 421         int err, ret;
 422
 423         if (unlikely(op->keylen != AES_KEYSIZE_128))
 424                 return fallback_blk_dec(desc, dst, src, nbytes);
 425
 426         blkcipher_walk_init(&walk, dst, src, nbytes);
 427         err = blkcipher_walk_virt(desc, &walk);
 428
 429         while ((nbytes = walk.nbytes)) {
 430                 op->src = walk.src.virt.addr,
 431                 op->dst = walk.dst.virt.addr;
 432                 op->mode = AES_MODE_ECB;
 433                 op->len = nbytes - (nbytes % AES_BLOCK_SIZE);
 434                 op->dir = AES_DIR_DECRYPT;
 435
 436                 ret = geode_aes_crypt(op);
 437                 nbytes -= ret;
 438                 err = blkcipher_walk_done(desc, &walk, nbytes);
 439         }
 440
 441         return err;
 442 }
 443
 444 static int
 445 geode_ecb_encrypt(struct blkcipher_desc *desc,
 446                   struct scatterlist *dst, struct scatterlist *src,
 447                   unsigned int nbytes)
 448 {
 449         struct geode_aes_op *op = crypto_blkcipher_ctx(desc->tfm);
 450         struct blkcipher_walk walk;
 451         int err, ret;
 452
 453         if (unlikely(op->keylen != AES_KEYSIZE_128))
 454                 return fallback_blk_enc(desc, dst, src, nbytes);
 455
 456         blkcipher_walk_init(&walk, dst, src, nbytes);
 457         err = blkcipher_walk_virt(desc, &walk);
 458
 459         while ((nbytes = walk.nbytes)) {
 460                 op->src = walk.src.virt.addr,
 461                 op->dst = walk.dst.virt.addr;
 462                 op->mode = AES_MODE_ECB;
 463                 op->len = nbytes - (nbytes % AES_BLOCK_SIZE);
 464                 op->dir = AES_DIR_ENCRYPT;
 465
 466                 ret = geode_aes_crypt(op);
 467                 nbytes -= ret;
 468                 ret =  blkcipher_walk_done(desc, &walk, nbytes);
 469         }
 470
 471         return err;
 472 }
 473
 474 static struct crypto_alg geode_ecb_alg = {
 475         .cra_name                       =       "ecb(aes)",
 476         .cra_driver_name        =       "ecb-aes-geode",
 477         .cra_priority           =       400,
 478         .cra_flags                      =       CRYPTO_ALG_TYPE_BLKCIPHER |
 479                                                 CRYPTO_ALG_KERN_DRIVER_ONLY |
 480                                                 CRYPTO_ALG_NEED_FALLBACK,
 481         .cra_init                       =       fallback_init_blk,
 482         .cra_exit                       =       fallback_exit_blk,
 483         .cra_blocksize          =       AES_BLOCK_SIZE,
 484         .cra_ctxsize            =       sizeof(struct geode_aes_op),
 485         .cra_alignmask          =       15,
 486         .cra_type                       =       &crypto_blkcipher_type,
 487         .cra_module                     =       THIS_MODULE,
 488         .cra_u                          =       {
 489                 .blkcipher      =       {
 490                         .min_keysize    =       AES_MIN_KEY_SIZE,
 491                         .max_keysize    =       AES_MAX_KEY_SIZE,
 492                         .setkey                 =       geode_setkey_blk,
 493                         .encrypt                =       geode_ecb_encrypt,
 494                         .decrypt                =       geode_ecb_decrypt,
 495                 }
 496         }
 497 };
 498
 499 static void geode_aes_remove(struct pci_dev *dev)
 500 {
 501         crypto_unregister_alg(&geode_alg);
 502         crypto_unregister_alg(&geode_ecb_alg);
 503         crypto_unregister_alg(&geode_cbc_alg);
 504
 505         pci_iounmap(dev, _iobase);
 506         _iobase = NULL;
 507
 508         pci_release_regions(dev);
 509         pci_disable_device(dev);
 510 }
 511
 512
 513 static int geode_aes_probe(struct pci_dev *dev, const struct pci_device_id *id)
 514 {
 515         int ret;
 516
 517         ret = pci_enable_device(dev);
 518         if (ret)
 519                 return ret;
 520
 521         ret = pci_request_regions(dev, "geode-aes");
 522         if (ret)
 523                 goto eenable;
 524
 525         _iobase = pci_iomap(dev, 0, 0);
 526
 527         if (_iobase == NULL) {
 528                 ret = -ENOMEM;
 529                 goto erequest;
 530         }
 531
 532         spin_lock_init(&lock);
 533
 534         /* Clear any pending activity */
 535         iowrite32(AES_INTR_PENDING | AES_INTR_MASK, _iobase + AES_INTR_REG);
 536
 537         ret = crypto_register_alg(&geode_alg);
 538         if (ret)
 539                 goto eiomap;
 540
 541         ret = crypto_register_alg(&geode_ecb_alg);
 542         if (ret)
 543                 goto ealg;
 544
 545         ret = crypto_register_alg(&geode_cbc_alg);
 546         if (ret)
 547                 goto eecb;
 548
 549         dev_notice(&dev->dev, "GEODE AES engine enabled.\n");
 550         return 0;
 551
 552  eecb:
 553         crypto_unregister_alg(&geode_ecb_alg);
 554
 555  ealg:
 556         crypto_unregister_alg(&geode_alg);
 557
 558  eiomap:
 559         pci_iounmap(dev, _iobase);
 560
 561  erequest:
 562         pci_release_regions(dev);
 563
 564  eenable:
 565         pci_disable_device(dev);
 566
 567         dev_err(&dev->dev, "GEODE AES initialization failed.\n");
 568         return ret;
 569 }
 570
 571 static struct pci_device_id geode_aes_tbl[] = {
 572         { PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_LX_AES), },
 573         { 0, }
 574 };
 575
 576 MODULE_DEVICE_TABLE(pci, geode_aes_tbl);
 577
 578 static struct pci_driver geode_aes_driver = {
 579         .name = "Geode LX AES",
 580         .id_table = geode_aes_tbl,
 581         .probe = geode_aes_probe,
 582         .remove = geode_aes_remove,
 583 };
 584
 585 module_pci_driver(geode_aes_driver);
 586
 587 MODULE_AUTHOR("Advanced Micro Devices, Inc.");
 588 MODULE_DESCRIPTION("Geode LX Hardware AES driver");
 589 MODULE_LICENSE("GPL");