arch/arm64/crypto/aes-glue.c

   1 /*
   2  * linux/arch/arm64/crypto/aes-glue.c - wrapper code for ARMv8 AES
   3  *
   4  * Copyright (C) 2013 Linaro Ltd <ard.biesheuvel@linaro.org>
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 as
   8  * published by the Free Software Foundation.
   9  */
  10
  11 #include <asm/neon.h>
  12 #include <asm/hwcap.h>
  13 #include <crypto/aes.h>
  14 #include <crypto/ablk_helper.h>
  15 #include <crypto/algapi.h>
  16 #include <linux/module.h>
  17 #include <linux/cpufeature.h>
  18
  19 #include "aes-ce-setkey.h"
  20
  21 #ifdef USE_V8_CRYPTO_EXTENSIONS
  22 #define MODE                    "ce"
  23 #define PRIO                    300
  24 #define aes_setkey              ce_aes_setkey
  25 #define aes_expandkey           ce_aes_expandkey
  26 #define aes_ecb_encrypt         ce_aes_ecb_encrypt
  27 #define aes_ecb_decrypt         ce_aes_ecb_decrypt
  28 #define aes_cbc_encrypt         ce_aes_cbc_encrypt
  29 #define aes_cbc_decrypt         ce_aes_cbc_decrypt
  30 #define aes_ctr_encrypt         ce_aes_ctr_encrypt
  31 #define aes_xts_encrypt         ce_aes_xts_encrypt
  32 #define aes_xts_decrypt         ce_aes_xts_decrypt
  33 MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 Crypto Extensions");
  34 #else
  35 #define MODE                    "neon"
  36 #define PRIO                    200
  37 #define aes_setkey              crypto_aes_set_key
  38 #define aes_expandkey           crypto_aes_expand_key
  39 #define aes_ecb_encrypt         neon_aes_ecb_encrypt
  40 #define aes_ecb_decrypt         neon_aes_ecb_decrypt
  41 #define aes_cbc_encrypt         neon_aes_cbc_encrypt
  42 #define aes_cbc_decrypt         neon_aes_cbc_decrypt
  43 #define aes_ctr_encrypt         neon_aes_ctr_encrypt
  44 #define aes_xts_encrypt         neon_aes_xts_encrypt
  45 #define aes_xts_decrypt         neon_aes_xts_decrypt
  46 MODULE_DESCRIPTION("AES-ECB/CBC/CTR/XTS using ARMv8 NEON");
  47 MODULE_ALIAS_CRYPTO("ecb(aes)");
  48 MODULE_ALIAS_CRYPTO("cbc(aes)");
  49 MODULE_ALIAS_CRYPTO("ctr(aes)");
  50 MODULE_ALIAS_CRYPTO("xts(aes)");
  51 #endif
  52
  53 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
  54 MODULE_LICENSE("GPL v2");
  55
  56 /* defined in aes-modes.S */
  57 asmlinkage void aes_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[],
  58                                 int rounds, int blocks, int first);
  59 asmlinkage void aes_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[],
  60                                 int rounds, int blocks, int first);
  61
  62 asmlinkage void aes_cbc_encrypt(u8 out[], u8 const in[], u8 const rk[],
  63                                 int rounds, int blocks, u8 iv[], int first);
  64 asmlinkage void aes_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[],
  65                                 int rounds, int blocks, u8 iv[], int first);
  66
  67 asmlinkage void aes_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[],
  68                                 int rounds, int blocks, u8 ctr[], int first);
  69
  70 asmlinkage void aes_xts_encrypt(u8 out[], u8 const in[], u8 const rk1[],
  71                                 int rounds, int blocks, u8 const rk2[], u8 iv[],
  72                                 int first);
  73 asmlinkage void aes_xts_decrypt(u8 out[], u8 const in[], u8 const rk1[],
  74                                 int rounds, int blocks, u8 const rk2[], u8 iv[],
  75                                 int first);
  76
  77 struct crypto_aes_xts_ctx {
  78         struct crypto_aes_ctx key1;
  79         struct crypto_aes_ctx __aligned(8) key2;
  80 };
  81
  82 static int xts_set_key(struct crypto_tfm *tfm, const u8 *in_key,
  83                        unsigned int key_len)
  84 {
  85         struct crypto_aes_xts_ctx *ctx = crypto_tfm_ctx(tfm);
  86         int ret;
  87
  88         ret = xts_check_key(tfm, in_key, key_len);
  89         if (ret)
  90                 return ret;
  91
  92         ret = aes_expandkey(&ctx->key1, in_key, key_len / 2);
  93         if (!ret)
  94                 ret = aes_expandkey(&ctx->key2, &in_key[key_len / 2],
  95                                     key_len / 2);
  96         if (!ret)
  97                 return 0;
  98
  99         tfm->crt_flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
 100         return -EINVAL;
 101 }
 102
 103 static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 104                        struct scatterlist *src, unsigned int nbytes)
 105 {
 106         struct crypto_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 107         int err, first, rounds = 6 + ctx->key_length / 4;
 108         struct blkcipher_walk walk;
 109         unsigned int blocks;
 110
 111         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 112         blkcipher_walk_init(&walk, dst, src, nbytes);
 113         err = blkcipher_walk_virt(desc, &walk);
 114
 115         kernel_neon_begin();
 116         for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
 117                 aes_ecb_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
 118                                 (u8 *)ctx->key_enc, rounds, blocks, first);
 119                 err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE);
 120         }
 121         kernel_neon_end();
 122         return err;
 123 }
 124
 125 static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 126                        struct scatterlist *src, unsigned int nbytes)
 127 {
 128         struct crypto_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 129         int err, first, rounds = 6 + ctx->key_length / 4;
 130         struct blkcipher_walk walk;
 131         unsigned int blocks;
 132
 133         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 134         blkcipher_walk_init(&walk, dst, src, nbytes);
 135         err = blkcipher_walk_virt(desc, &walk);
 136
 137         kernel_neon_begin();
 138         for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
 139                 aes_ecb_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
 140                                 (u8 *)ctx->key_dec, rounds, blocks, first);
 141                 err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE);
 142         }
 143         kernel_neon_end();
 144         return err;
 145 }
 146
 147 static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 148                        struct scatterlist *src, unsigned int nbytes)
 149 {
 150         struct crypto_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 151         int err, first, rounds = 6 + ctx->key_length / 4;
 152         struct blkcipher_walk walk;
 153         unsigned int blocks;
 154
 155         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 156         blkcipher_walk_init(&walk, dst, src, nbytes);
 157         err = blkcipher_walk_virt(desc, &walk);
 158
 159         kernel_neon_begin();
 160         for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
 161                 aes_cbc_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
 162                                 (u8 *)ctx->key_enc, rounds, blocks, walk.iv,
 163                                 first);
 164                 err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE);
 165         }
 166         kernel_neon_end();
 167         return err;
 168 }
 169
 170 static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 171                        struct scatterlist *src, unsigned int nbytes)
 172 {
 173         struct crypto_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 174         int err, first, rounds = 6 + ctx->key_length / 4;
 175         struct blkcipher_walk walk;
 176         unsigned int blocks;
 177
 178         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 179         blkcipher_walk_init(&walk, dst, src, nbytes);
 180         err = blkcipher_walk_virt(desc, &walk);
 181
 182         kernel_neon_begin();
 183         for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
 184                 aes_cbc_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
 185                                 (u8 *)ctx->key_dec, rounds, blocks, walk.iv,
 186                                 first);
 187                 err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE);
 188         }
 189         kernel_neon_end();
 190         return err;
 191 }
 192
 193 static int ctr_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 194                        struct scatterlist *src, unsigned int nbytes)
 195 {
 196         struct crypto_aes_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 197         int err, first, rounds = 6 + ctx->key_length / 4;
 198         struct blkcipher_walk walk;
 199         int blocks;
 200
 201         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 202         blkcipher_walk_init(&walk, dst, src, nbytes);
 203         err = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE);
 204
 205         first = 1;
 206         kernel_neon_begin();
 207         while ((blocks = (walk.nbytes / AES_BLOCK_SIZE))) {
 208                 aes_ctr_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
 209                                 (u8 *)ctx->key_enc, rounds, blocks, walk.iv,
 210                                 first);
 211                 first = 0;
 212                 nbytes -= blocks * AES_BLOCK_SIZE;
 213                 if (nbytes && nbytes == walk.nbytes % AES_BLOCK_SIZE)
 214                         break;
 215                 err = blkcipher_walk_done(desc, &walk,
 216                                           walk.nbytes % AES_BLOCK_SIZE);
 217         }
 218         if (nbytes) {
 219                 u8 *tdst = walk.dst.virt.addr + blocks * AES_BLOCK_SIZE;
 220                 u8 *tsrc = walk.src.virt.addr + blocks * AES_BLOCK_SIZE;
 221                 u8 __aligned(8) tail[AES_BLOCK_SIZE];
 222
 223                 /*
 224                  * Minimum alignment is 8 bytes, so if nbytes is <= 8, we need
 225                  * to tell aes_ctr_encrypt() to only read half a block.
 226                  */
 227                 blocks = (nbytes <= 8) ? -1 : 1;
 228
 229                 aes_ctr_encrypt(tail, tsrc, (u8 *)ctx->key_enc, rounds,
 230                                 blocks, walk.iv, first);
 231                 memcpy(tdst, tail, nbytes);
 232                 err = blkcipher_walk_done(desc, &walk, 0);
 233         }
 234         kernel_neon_end();
 235
 236         return err;
 237 }
 238
 239 static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 240                        struct scatterlist *src, unsigned int nbytes)
 241 {
 242         struct crypto_aes_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 243         int err, first, rounds = 6 + ctx->key1.key_length / 4;
 244         struct blkcipher_walk walk;
 245         unsigned int blocks;
 246
 247         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 248         blkcipher_walk_init(&walk, dst, src, nbytes);
 249         err = blkcipher_walk_virt(desc, &walk);
 250
 251         kernel_neon_begin();
 252         for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
 253                 aes_xts_encrypt(walk.dst.virt.addr, walk.src.virt.addr,
 254                                 (u8 *)ctx->key1.key_enc, rounds, blocks,
 255                                 (u8 *)ctx->key2.key_enc, walk.iv, first);
 256                 err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE);
 257         }
 258         kernel_neon_end();
 259
 260         return err;
 261 }
 262
 263 static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
 264                        struct scatterlist *src, unsigned int nbytes)
 265 {
 266         struct crypto_aes_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
 267         int err, first, rounds = 6 + ctx->key1.key_length / 4;
 268         struct blkcipher_walk walk;
 269         unsigned int blocks;
 270
 271         desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
 272         blkcipher_walk_init(&walk, dst, src, nbytes);
 273         err = blkcipher_walk_virt(desc, &walk);
 274
 275         kernel_neon_begin();
 276         for (first = 1; (blocks = (walk.nbytes / AES_BLOCK_SIZE)); first = 0) {
 277                 aes_xts_decrypt(walk.dst.virt.addr, walk.src.virt.addr,
 278                                 (u8 *)ctx->key1.key_dec, rounds, blocks,
 279                                 (u8 *)ctx->key2.key_enc, walk.iv, first);
 280                 err = blkcipher_walk_done(desc, &walk, walk.nbytes % AES_BLOCK_SIZE);
 281         }
 282         kernel_neon_end();
 283
 284         return err;
 285 }
 286
 287 static struct crypto_alg aes_algs[] = { {
 288         .cra_name               = "__ecb-aes-" MODE,
 289         .cra_driver_name        = "__driver-ecb-aes-" MODE,
 290         .cra_priority           = 0,
 291         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER |
 292                                   CRYPTO_ALG_INTERNAL,
 293         .cra_blocksize          = AES_BLOCK_SIZE,
 294         .cra_ctxsize            = sizeof(struct crypto_aes_ctx),
 295         .cra_alignmask          = 7,
 296         .cra_type               = &crypto_blkcipher_type,
 297         .cra_module             = THIS_MODULE,
 298         .cra_blkcipher = {
 299                 .min_keysize    = AES_MIN_KEY_SIZE,
 300                 .max_keysize    = AES_MAX_KEY_SIZE,
 301                 .ivsize         = AES_BLOCK_SIZE,
 302                 .setkey         = aes_setkey,
 303                 .encrypt        = ecb_encrypt,
 304                 .decrypt        = ecb_decrypt,
 305         },
 306 }, {
 307         .cra_name               = "__cbc-aes-" MODE,
 308         .cra_driver_name        = "__driver-cbc-aes-" MODE,
 309         .cra_priority           = 0,
 310         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER |
 311                                   CRYPTO_ALG_INTERNAL,
 312         .cra_blocksize          = AES_BLOCK_SIZE,
 313         .cra_ctxsize            = sizeof(struct crypto_aes_ctx),
 314         .cra_alignmask          = 7,
 315         .cra_type               = &crypto_blkcipher_type,
 316         .cra_module             = THIS_MODULE,
 317         .cra_blkcipher = {
 318                 .min_keysize    = AES_MIN_KEY_SIZE,
 319                 .max_keysize    = AES_MAX_KEY_SIZE,
 320                 .ivsize         = AES_BLOCK_SIZE,
 321                 .setkey         = aes_setkey,
 322                 .encrypt        = cbc_encrypt,
 323                 .decrypt        = cbc_decrypt,
 324         },
 325 }, {
 326         .cra_name               = "__ctr-aes-" MODE,
 327         .cra_driver_name        = "__driver-ctr-aes-" MODE,
 328         .cra_priority           = 0,
 329         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER |
 330                                   CRYPTO_ALG_INTERNAL,
 331         .cra_blocksize          = 1,
 332         .cra_ctxsize            = sizeof(struct crypto_aes_ctx),
 333         .cra_alignmask          = 7,
 334         .cra_type               = &crypto_blkcipher_type,
 335         .cra_module             = THIS_MODULE,
 336         .cra_blkcipher = {
 337                 .min_keysize    = AES_MIN_KEY_SIZE,
 338                 .max_keysize    = AES_MAX_KEY_SIZE,
 339                 .ivsize         = AES_BLOCK_SIZE,
 340                 .setkey         = aes_setkey,
 341                 .encrypt        = ctr_encrypt,
 342                 .decrypt        = ctr_encrypt,
 343         },
 344 }, {
 345         .cra_name               = "__xts-aes-" MODE,
 346         .cra_driver_name        = "__driver-xts-aes-" MODE,
 347         .cra_priority           = 0,
 348         .cra_flags              = CRYPTO_ALG_TYPE_BLKCIPHER |
 349                                   CRYPTO_ALG_INTERNAL,
 350         .cra_blocksize          = AES_BLOCK_SIZE,
 351         .cra_ctxsize            = sizeof(struct crypto_aes_xts_ctx),
 352         .cra_alignmask          = 7,
 353         .cra_type               = &crypto_blkcipher_type,
 354         .cra_module             = THIS_MODULE,
 355         .cra_blkcipher = {
 356                 .min_keysize    = 2 * AES_MIN_KEY_SIZE,
 357                 .max_keysize    = 2 * AES_MAX_KEY_SIZE,
 358                 .ivsize         = AES_BLOCK_SIZE,
 359                 .setkey         = xts_set_key,
 360                 .encrypt        = xts_encrypt,
 361                 .decrypt        = xts_decrypt,
 362         },
 363 }, {
 364         .cra_name               = "ecb(aes)",
 365         .cra_driver_name        = "ecb-aes-" MODE,
 366         .cra_priority           = PRIO,
 367         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
 368         .cra_blocksize          = AES_BLOCK_SIZE,
 369         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 370         .cra_alignmask          = 7,
 371         .cra_type               = &crypto_ablkcipher_type,
 372         .cra_module             = THIS_MODULE,
 373         .cra_init               = ablk_init,
 374         .cra_exit               = ablk_exit,
 375         .cra_ablkcipher = {
 376                 .min_keysize    = AES_MIN_KEY_SIZE,
 377                 .max_keysize    = AES_MAX_KEY_SIZE,
 378                 .ivsize         = AES_BLOCK_SIZE,
 379                 .setkey         = ablk_set_key,
 380                 .encrypt        = ablk_encrypt,
 381                 .decrypt        = ablk_decrypt,
 382         }
 383 }, {
 384         .cra_name               = "cbc(aes)",
 385         .cra_driver_name        = "cbc-aes-" MODE,
 386         .cra_priority           = PRIO,
 387         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
 388         .cra_blocksize          = AES_BLOCK_SIZE,
 389         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 390         .cra_alignmask          = 7,
 391         .cra_type               = &crypto_ablkcipher_type,
 392         .cra_module             = THIS_MODULE,
 393         .cra_init               = ablk_init,
 394         .cra_exit               = ablk_exit,
 395         .cra_ablkcipher = {
 396                 .min_keysize    = AES_MIN_KEY_SIZE,
 397                 .max_keysize    = AES_MAX_KEY_SIZE,
 398                 .ivsize         = AES_BLOCK_SIZE,
 399                 .setkey         = ablk_set_key,
 400                 .encrypt        = ablk_encrypt,
 401                 .decrypt        = ablk_decrypt,
 402         }
 403 }, {
 404         .cra_name               = "ctr(aes)",
 405         .cra_driver_name        = "ctr-aes-" MODE,
 406         .cra_priority           = PRIO,
 407         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
 408         .cra_blocksize          = 1,
 409         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 410         .cra_alignmask          = 7,
 411         .cra_type               = &crypto_ablkcipher_type,
 412         .cra_module             = THIS_MODULE,
 413         .cra_init               = ablk_init,
 414         .cra_exit               = ablk_exit,
 415         .cra_ablkcipher = {
 416                 .min_keysize    = AES_MIN_KEY_SIZE,
 417                 .max_keysize    = AES_MAX_KEY_SIZE,
 418                 .ivsize         = AES_BLOCK_SIZE,
 419                 .setkey         = ablk_set_key,
 420                 .encrypt        = ablk_encrypt,
 421                 .decrypt        = ablk_decrypt,
 422         }
 423 }, {
 424         .cra_name               = "xts(aes)",
 425         .cra_driver_name        = "xts-aes-" MODE,
 426         .cra_priority           = PRIO,
 427         .cra_flags              = CRYPTO_ALG_TYPE_ABLKCIPHER|CRYPTO_ALG_ASYNC,
 428         .cra_blocksize          = AES_BLOCK_SIZE,
 429         .cra_ctxsize            = sizeof(struct async_helper_ctx),
 430         .cra_alignmask          = 7,
 431         .cra_type               = &crypto_ablkcipher_type,
 432         .cra_module             = THIS_MODULE,
 433         .cra_init               = ablk_init,
 434         .cra_exit               = ablk_exit,
 435         .cra_ablkcipher = {
 436                 .min_keysize    = 2 * AES_MIN_KEY_SIZE,
 437                 .max_keysize    = 2 * AES_MAX_KEY_SIZE,
 438                 .ivsize         = AES_BLOCK_SIZE,
 439                 .setkey         = ablk_set_key,
 440                 .encrypt        = ablk_encrypt,
 441                 .decrypt        = ablk_decrypt,
 442         }
 443 } };
 444
 445 static int __init aes_init(void)
 446 {
 447         return crypto_register_algs(aes_algs, ARRAY_SIZE(aes_algs));
 448 }
 449
 450 static void __exit aes_exit(void)
 451 {
 452         crypto_unregister_algs(aes_algs, ARRAY_SIZE(aes_algs));
 453 }
 454
 455 #ifdef USE_V8_CRYPTO_EXTENSIONS
 456 module_cpu_feature_match(AES, aes_init);
 457 #else
 458 module_init(aes_init);
 459 #endif
 460 module_exit(aes_exit);