crypto: arm64/aes-neon-blk - tweak performance for low end cores

[mirror_ubuntu-focal-kernel.git] / arch / arm64 / crypto / aes-glue.c

diff --git a/arch/arm64/crypto/aes-glue.c b/arch/arm64/crypto/aes-glue.c
index 4e3f8adb179397115be300b998beefc645be1092..055bc3f61138843f337754e0def2e68abc7e47bc 100644 (file)
--- a/arch/arm64/crypto/aes-glue.c
+++ b/arch/arm64/crypto/aes-glue.c
@@ -215,14 +215,15 @@ static int ctr_encrypt(struct skcipher_request *req)
                u8 *tsrc = walk.src.virt.addr;
 
                /*
-                * Minimum alignment is 8 bytes, so if nbytes is <= 8, we need
-                * to tell aes_ctr_encrypt() to only read half a block.
+                * Tell aes_ctr_encrypt() to process a tail block.
                 */
-               blocks = (nbytes <= 8) ? -1 : 1;
+               blocks = -1;
 
-               aes_ctr_encrypt(tail, tsrc, (u8 *)ctx->key_enc, rounds,
+               aes_ctr_encrypt(tail, NULL, (u8 *)ctx->key_enc, rounds,
                                blocks, walk.iv, first);
-               memcpy(tdst, tail, nbytes);
+               if (tdst != tsrc)
+                       memcpy(tdst, tsrc, nbytes);
+               crypto_xor(tdst, tail, nbytes);
                err = skcipher_walk_done(&walk, 0);
        }
        kernel_neon_end();
@@ -282,7 +283,6 @@ static struct skcipher_alg aes_algs[] = { {
                .cra_flags              = CRYPTO_ALG_INTERNAL,
                .cra_blocksize          = AES_BLOCK_SIZE,
                .cra_ctxsize            = sizeof(struct crypto_aes_ctx),
-               .cra_alignmask          = 7,
                .cra_module             = THIS_MODULE,
        },
        .min_keysize    = AES_MIN_KEY_SIZE,
@@ -298,7 +298,6 @@ static struct skcipher_alg aes_algs[] = { {
                .cra_flags              = CRYPTO_ALG_INTERNAL,
                .cra_blocksize          = AES_BLOCK_SIZE,
                .cra_ctxsize            = sizeof(struct crypto_aes_ctx),
-               .cra_alignmask          = 7,
                .cra_module             = THIS_MODULE,
        },
        .min_keysize    = AES_MIN_KEY_SIZE,
@@ -315,7 +314,22 @@ static struct skcipher_alg aes_algs[] = { {
                .cra_flags              = CRYPTO_ALG_INTERNAL,
                .cra_blocksize          = 1,
                .cra_ctxsize            = sizeof(struct crypto_aes_ctx),
-               .cra_alignmask          = 7,
+               .cra_module             = THIS_MODULE,
+       },
+       .min_keysize    = AES_MIN_KEY_SIZE,
+       .max_keysize    = AES_MAX_KEY_SIZE,
+       .ivsize         = AES_BLOCK_SIZE,
+       .chunksize      = AES_BLOCK_SIZE,
+       .setkey         = skcipher_aes_setkey,
+       .encrypt        = ctr_encrypt,
+       .decrypt        = ctr_encrypt,
+}, {
+       .base = {
+               .cra_name               = "ctr(aes)",
+               .cra_driver_name        = "ctr-aes-" MODE,
+               .cra_priority           = PRIO - 1,
+               .cra_blocksize          = 1,
+               .cra_ctxsize            = sizeof(struct crypto_aes_ctx),
                .cra_module             = THIS_MODULE,
        },
        .min_keysize    = AES_MIN_KEY_SIZE,
@@ -333,7 +347,6 @@ static struct skcipher_alg aes_algs[] = { {
                .cra_flags              = CRYPTO_ALG_INTERNAL,
                .cra_blocksize          = AES_BLOCK_SIZE,
                .cra_ctxsize            = sizeof(struct crypto_aes_xts_ctx),
-               .cra_alignmask          = 7,
                .cra_module             = THIS_MODULE,
        },
        .min_keysize    = 2 * AES_MIN_KEY_SIZE,
@@ -350,8 +363,9 @@ static void aes_exit(void)
 {
        int i;
 
-       for (i = 0; i < ARRAY_SIZE(aes_simd_algs) && aes_simd_algs[i]; i++)
-               simd_skcipher_free(aes_simd_algs[i]);
+       for (i = 0; i < ARRAY_SIZE(aes_simd_algs); i++)
+               if (aes_simd_algs[i])
+                       simd_skcipher_free(aes_simd_algs[i]);
 
        crypto_unregister_skciphers(aes_algs, ARRAY_SIZE(aes_algs));
 }
@@ -370,6 +384,9 @@ static int __init aes_init(void)
                return err;
 
        for (i = 0; i < ARRAY_SIZE(aes_algs); i++) {
+               if (!(aes_algs[i].base.cra_flags & CRYPTO_ALG_INTERNAL))
+                       continue;
+
                algname = aes_algs[i].base.cra_name + 2;
                drvname = aes_algs[i].base.cra_driver_name + 2;
                basename = aes_algs[i].base.cra_driver_name;
@@ -392,5 +409,7 @@ unregister_simds:
 module_cpu_feature_match(AES, aes_init);
 #else
 module_init(aes_init);
+EXPORT_SYMBOL(neon_aes_ecb_encrypt);
+EXPORT_SYMBOL(neon_aes_cbc_encrypt);
 #endif
 module_exit(aes_exit);