crypto: arm/aes-neonbs - implement ciphertext stealing for XTS

Update the AES-XTS implementation based on NEON instructions so that it
can deal with inputs whose size is not a multiple of the cipher block
size. This is part of the original XTS specification, but was never
implemented before in the Linux kernel.

Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>

diff --git a/arch/arm/crypto/aes-neonbs-core.S b/arch/arm/crypto/aes-neonbs-core.S
index bb75918e4984ac54edfde844ae8d4653063a266c..cfaed4e67535f5ef7453c76336b474cd1923868c 100644 (file)
--- a/arch/arm/crypto/aes-neonbs-core.S
+++ b/arch/arm/crypto/aes-neonbs-core.S
@@ -889,9 +889,9 @@ ENDPROC(aesbs_ctr_encrypt)
 
        /*
         * aesbs_xts_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
-        *                   int blocks, u8 iv[])
+        *                   int blocks, u8 iv[], int reorder_last_tweak)
         * aesbs_xts_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds,
-        *                   int blocks, u8 iv[])
+        *                   int blocks, u8 iv[], int reorder_last_tweak)
         */
 __xts_prepare8:
        vld1.8          {q14}, [r7]             // load iv
@@ -944,17 +944,25 @@ __xts_prepare8:
 
        vld1.8          {q7}, [r1]!
        next_tweak      q14, q12, q15, q13
-       veor            q7, q7, q12
+THUMB( itt             le              )
+       W(cmple)        r8, #0
+       ble             1f
+0:     veor            q7, q7, q12
        vst1.8          {q12}, [r4, :128]
 
-0:     vst1.8          {q14}, [r7]             // store next iv
+       vst1.8          {q14}, [r7]             // store next iv
        bx              lr
+
+1:     vswp            q12, q14
+       b               0b
 ENDPROC(__xts_prepare8)
 
        .macro          __xts_crypt, do8, o0, o1, o2, o3, o4, o5, o6, o7
        push            {r4-r8, lr}
        mov             r5, sp                  // preserve sp
        ldrd            r6, r7, [sp, #24]       // get blocks and iv args
+       ldr             r8, [sp, #32]           // reorder final tweak?
+       rsb             r8, r8, #1
        sub             ip, sp, #128            // make room for 8x tweak
        bic             ip, ip, #0xf            // align sp to 16 bytes
        mov             sp, ip

diff --git a/arch/arm/crypto/aes-neonbs-glue.c b/arch/arm/crypto/aes-neonbs-glue.c
index 9000d0796d5ec3b31f218a2e392d62c8f2c3378c..e85839a8aaeb8b35d6f4f5e0ac804da1f7543c9d 100644 (file)
--- a/arch/arm/crypto/aes-neonbs-glue.c
+++ b/arch/arm/crypto/aes-neonbs-glue.c
@@ -12,6 +12,7 @@
 #include <crypto/ctr.h>
 #include <crypto/internal/simd.h>
 #include <crypto/internal/skcipher.h>
+#include <crypto/scatterwalk.h>
 #include <crypto/xts.h>
 #include <linux/module.h>
 
@@ -37,9 +38,9 @@ asmlinkage void aesbs_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[],
                                  int rounds, int blocks, u8 ctr[], u8 final[]);
 
 asmlinkage void aesbs_xts_encrypt(u8 out[], u8 const in[], u8 const rk[],
-                                 int rounds, int blocks, u8 iv[]);
+                                 int rounds, int blocks, u8 iv[], int);
 asmlinkage void aesbs_xts_decrypt(u8 out[], u8 const in[], u8 const rk[],
-                                 int rounds, int blocks, u8 iv[]);
+                                 int rounds, int blocks, u8 iv[], int);
 
 struct aesbs_ctx {
        int     rounds;
@@ -53,6 +54,7 @@ struct aesbs_cbc_ctx {
 
 struct aesbs_xts_ctx {
        struct aesbs_ctx        key;
+       struct crypto_cipher    *cts_tfm;
        struct crypto_cipher    *tweak_tfm;
 };
 
@@ -291,6 +293,9 @@ static int aesbs_xts_setkey(struct crypto_skcipher *tfm, const u8 *in_key,
                return err;
 
        key_len /= 2;
+       err = crypto_cipher_setkey(ctx->cts_tfm, in_key, key_len);
+       if (err)
+               return err;
        err = crypto_cipher_setkey(ctx->tweak_tfm, in_key + key_len, key_len);
        if (err)
                return err;
@@ -302,7 +307,13 @@ static int xts_init(struct crypto_tfm *tfm)
 {
        struct aesbs_xts_ctx *ctx = crypto_tfm_ctx(tfm);
 
+       ctx->cts_tfm = crypto_alloc_cipher("aes", 0, 0);
+       if (IS_ERR(ctx->cts_tfm))
+               return PTR_ERR(ctx->cts_tfm);
+
        ctx->tweak_tfm = crypto_alloc_cipher("aes", 0, 0);
+       if (IS_ERR(ctx->tweak_tfm))
+               crypto_free_cipher(ctx->cts_tfm);
 
        return PTR_ERR_OR_ZERO(ctx->tweak_tfm);
 }
@@ -312,17 +323,34 @@ static void xts_exit(struct crypto_tfm *tfm)
        struct aesbs_xts_ctx *ctx = crypto_tfm_ctx(tfm);
 
        crypto_free_cipher(ctx->tweak_tfm);
+       crypto_free_cipher(ctx->cts_tfm);
 }
 
-static int __xts_crypt(struct skcipher_request *req,
+static int __xts_crypt(struct skcipher_request *req, bool encrypt,
                       void (*fn)(u8 out[], u8 const in[], u8 const rk[],
-                                 int rounds, int blocks, u8 iv[]))
+                                 int rounds, int blocks, u8 iv[], int))
 {
        struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
        struct aesbs_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
+       int tail = req->cryptlen % AES_BLOCK_SIZE;
+       struct skcipher_request subreq;
+       u8 buf[2 * AES_BLOCK_SIZE];
        struct skcipher_walk walk;
        int err;
 
+       if (req->cryptlen < AES_BLOCK_SIZE)
+               return -EINVAL;
+
+       if (unlikely(tail)) {
+               skcipher_request_set_tfm(&subreq, tfm);
+               skcipher_request_set_callback(&subreq,
+                                             skcipher_request_flags(req),
+                                             NULL, NULL);
+               skcipher_request_set_crypt(&subreq, req->src, req->dst,
+                                          req->cryptlen - tail, req->iv);
+               req = &subreq;
+       }
+
        err = skcipher_walk_virt(&walk, req, true);
        if (err)
                return err;
@@ -331,30 +359,53 @@ static int __xts_crypt(struct skcipher_request *req,
 
        while (walk.nbytes >= AES_BLOCK_SIZE) {
                unsigned int blocks = walk.nbytes / AES_BLOCK_SIZE;
+               int reorder_last_tweak = !encrypt && tail > 0;
 
-               if (walk.nbytes < walk.total)
+               if (walk.nbytes < walk.total) {
                        blocks = round_down(blocks,
                                            walk.stride / AES_BLOCK_SIZE);
+                       reorder_last_tweak = 0;
+               }
 
                kernel_neon_begin();
                fn(walk.dst.virt.addr, walk.src.virt.addr, ctx->key.rk,
-                  ctx->key.rounds, blocks, walk.iv);
+                  ctx->key.rounds, blocks, walk.iv, reorder_last_tweak);
                kernel_neon_end();
                err = skcipher_walk_done(&walk,
                                         walk.nbytes - blocks * AES_BLOCK_SIZE);
        }
 
-       return err;
+       if (err || likely(!tail))
+               return err;
+
+       /* handle ciphertext stealing */
+       scatterwalk_map_and_copy(buf, req->dst, req->cryptlen - AES_BLOCK_SIZE,
+                                AES_BLOCK_SIZE, 0);
+       memcpy(buf + AES_BLOCK_SIZE, buf, tail);
+       scatterwalk_map_and_copy(buf, req->src, req->cryptlen, tail, 0);
+
+       crypto_xor(buf, req->iv, AES_BLOCK_SIZE);
+
+       if (encrypt)
+               crypto_cipher_encrypt_one(ctx->cts_tfm, buf, buf);
+       else
+               crypto_cipher_decrypt_one(ctx->cts_tfm, buf, buf);
+
+       crypto_xor(buf, req->iv, AES_BLOCK_SIZE);
+
+       scatterwalk_map_and_copy(buf, req->dst, req->cryptlen - AES_BLOCK_SIZE,
+                                AES_BLOCK_SIZE + tail, 1);
+       return 0;
 }
 
 static int xts_encrypt(struct skcipher_request *req)
 {
-       return __xts_crypt(req, aesbs_xts_encrypt);
+       return __xts_crypt(req, true, aesbs_xts_encrypt);
 }
 
 static int xts_decrypt(struct skcipher_request *req)
 {
-       return __xts_crypt(req, aesbs_xts_decrypt);
+       return __xts_crypt(req, false, aesbs_xts_decrypt);
 }
 
 static struct skcipher_alg aes_algs[] = { {