drivers/crypto/omap-aes-gcm.c

   1 /*
   2  * Cryptographic API.
   3  *
   4  * Support for OMAP AES GCM HW acceleration.
   5  *
   6  * Copyright (c) 2016 Texas Instruments Incorporated
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License version 2 as published
  10  * by the Free Software Foundation.
  11  *
  12  */
  13
  14 #include <linux/errno.h>
  15 #include <linux/scatterlist.h>
  16 #include <linux/dma-mapping.h>
  17 #include <linux/dmaengine.h>
  18 #include <linux/omap-dma.h>
  19 #include <linux/interrupt.h>
  20 #include <crypto/aes.h>
  21 #include <crypto/scatterwalk.h>
  22 #include <crypto/skcipher.h>
  23 #include <crypto/internal/aead.h>
  24
  25 #include "omap-crypto.h"
  26 #include "omap-aes.h"
  27
  28 static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd,
  29                                      struct aead_request *req);
  30
  31 static void omap_aes_gcm_finish_req(struct omap_aes_dev *dd, int ret)
  32 {
  33         struct aead_request *req = dd->aead_req;
  34
  35         dd->flags &= ~FLAGS_BUSY;
  36         dd->in_sg = NULL;
  37         dd->out_sg = NULL;
  38
  39         req->base.complete(&req->base, ret);
  40 }
  41
  42 static void omap_aes_gcm_done_task(struct omap_aes_dev *dd)
  43 {
  44         u8 *tag;
  45         int alen, clen, i, ret = 0, nsg;
  46         struct omap_aes_reqctx *rctx;
  47
  48         alen = ALIGN(dd->assoc_len, AES_BLOCK_SIZE);
  49         clen = ALIGN(dd->total, AES_BLOCK_SIZE);
  50         rctx = aead_request_ctx(dd->aead_req);
  51
  52         nsg = !!(dd->assoc_len && dd->total);
  53
  54         dma_sync_sg_for_device(dd->dev, dd->out_sg, dd->out_sg_len,
  55                                DMA_FROM_DEVICE);
  56         dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE);
  57         dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len, DMA_FROM_DEVICE);
  58         omap_aes_crypt_dma_stop(dd);
  59
  60         omap_crypto_cleanup(dd->out_sg, dd->orig_out,
  61                             dd->aead_req->assoclen, dd->total,
  62                             FLAGS_OUT_DATA_ST_SHIFT, dd->flags);
  63
  64         if (dd->flags & FLAGS_ENCRYPT)
  65                 scatterwalk_map_and_copy(rctx->auth_tag,
  66                                          dd->aead_req->dst,
  67                                          dd->total + dd->aead_req->assoclen,
  68                                          dd->authsize, 1);
  69
  70         omap_crypto_cleanup(&dd->in_sgl[0], NULL, 0, alen,
  71                             FLAGS_ASSOC_DATA_ST_SHIFT, dd->flags);
  72
  73         omap_crypto_cleanup(&dd->in_sgl[nsg], NULL, 0, clen,
  74                             FLAGS_IN_DATA_ST_SHIFT, dd->flags);
  75
  76         if (!(dd->flags & FLAGS_ENCRYPT)) {
  77                 tag = (u8 *)rctx->auth_tag;
  78                 for (i = 0; i < dd->authsize; i++) {
  79                         if (tag[i]) {
  80                                 dev_err(dd->dev, "GCM decryption: Tag Message is wrong\n");
  81                                 ret = -EBADMSG;
  82                         }
  83                 }
  84         }
  85
  86         omap_aes_gcm_finish_req(dd, ret);
  87         omap_aes_gcm_handle_queue(dd, NULL);
  88 }
  89
  90 static int omap_aes_gcm_copy_buffers(struct omap_aes_dev *dd,
  91                                      struct aead_request *req)
  92 {
  93         int alen, clen, cryptlen, assoclen, ret;
  94         struct crypto_aead *aead = crypto_aead_reqtfm(req);
  95         unsigned int authlen = crypto_aead_authsize(aead);
  96         struct scatterlist *tmp, sg_arr[2];
  97         int nsg;
  98         u16 flags;
  99
 100         assoclen = req->assoclen;
 101         cryptlen = req->cryptlen;
 102
 103         if (dd->flags & FLAGS_RFC4106_GCM)
 104                 assoclen -= 8;
 105
 106         if (!(dd->flags & FLAGS_ENCRYPT))
 107                 cryptlen -= authlen;
 108
 109         alen = ALIGN(assoclen, AES_BLOCK_SIZE);
 110         clen = ALIGN(cryptlen, AES_BLOCK_SIZE);
 111
 112         nsg = !!(assoclen && cryptlen);
 113
 114         omap_aes_clear_copy_flags(dd);
 115
 116         sg_init_table(dd->in_sgl, nsg + 1);
 117         if (assoclen) {
 118                 tmp = req->src;
 119                 ret = omap_crypto_align_sg(&tmp, assoclen,
 120                                            AES_BLOCK_SIZE, dd->in_sgl,
 121                                            OMAP_CRYPTO_COPY_DATA |
 122                                            OMAP_CRYPTO_ZERO_BUF |
 123                                            OMAP_CRYPTO_FORCE_SINGLE_ENTRY,
 124                                            FLAGS_ASSOC_DATA_ST_SHIFT,
 125                                            &dd->flags);
 126         }
 127
 128         if (cryptlen) {
 129                 tmp = scatterwalk_ffwd(sg_arr, req->src, req->assoclen);
 130
 131                 ret = omap_crypto_align_sg(&tmp, cryptlen,
 132                                            AES_BLOCK_SIZE, &dd->in_sgl[nsg],
 133                                            OMAP_CRYPTO_COPY_DATA |
 134                                            OMAP_CRYPTO_ZERO_BUF |
 135                                            OMAP_CRYPTO_FORCE_SINGLE_ENTRY,
 136                                            FLAGS_IN_DATA_ST_SHIFT,
 137                                            &dd->flags);
 138         }
 139
 140         dd->in_sg = dd->in_sgl;
 141         dd->total = cryptlen;
 142         dd->assoc_len = assoclen;
 143         dd->authsize = authlen;
 144
 145         dd->out_sg = req->dst;
 146         dd->orig_out = req->dst;
 147
 148         dd->out_sg = scatterwalk_ffwd(sg_arr, req->dst, assoclen);
 149
 150         flags = 0;
 151         if (req->src == req->dst || dd->out_sg == sg_arr)
 152                 flags |= OMAP_CRYPTO_FORCE_COPY;
 153
 154         ret = omap_crypto_align_sg(&dd->out_sg, cryptlen,
 155                                    AES_BLOCK_SIZE, &dd->out_sgl,
 156                                    flags,
 157                                    FLAGS_OUT_DATA_ST_SHIFT, &dd->flags);
 158         if (ret)
 159                 return ret;
 160
 161         dd->in_sg_len = sg_nents_for_len(dd->in_sg, alen + clen);
 162         dd->out_sg_len = sg_nents_for_len(dd->out_sg, clen);
 163
 164         return 0;
 165 }
 166
 167 static void omap_aes_gcm_complete(struct crypto_async_request *req, int err)
 168 {
 169         struct omap_aes_gcm_result *res = req->data;
 170
 171         if (err == -EINPROGRESS)
 172                 return;
 173
 174         res->err = err;
 175         complete(&res->completion);
 176 }
 177
 178 static int do_encrypt_iv(struct aead_request *req, u32 *tag, u32 *iv)
 179 {
 180         struct scatterlist iv_sg, tag_sg;
 181         struct skcipher_request *sk_req;
 182         struct omap_aes_gcm_result result;
 183         struct omap_aes_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
 184         int ret = 0;
 185
 186         sk_req = skcipher_request_alloc(ctx->ctr, GFP_KERNEL);
 187         if (!sk_req) {
 188                 pr_err("skcipher: Failed to allocate request\n");
 189                 return -1;
 190         }
 191
 192         init_completion(&result.completion);
 193
 194         sg_init_one(&iv_sg, iv, AES_BLOCK_SIZE);
 195         sg_init_one(&tag_sg, tag, AES_BLOCK_SIZE);
 196         skcipher_request_set_callback(sk_req, CRYPTO_TFM_REQ_MAY_BACKLOG,
 197                                       omap_aes_gcm_complete, &result);
 198         ret = crypto_skcipher_setkey(ctx->ctr, (u8 *)ctx->key, ctx->keylen);
 199         skcipher_request_set_crypt(sk_req, &iv_sg, &tag_sg, AES_BLOCK_SIZE,
 200                                    NULL);
 201         ret = crypto_skcipher_encrypt(sk_req);
 202         switch (ret) {
 203         case 0:
 204                 break;
 205         case -EINPROGRESS:
 206         case -EBUSY:
 207                 ret = wait_for_completion_interruptible(&result.completion);
 208                 if (!ret) {
 209                         ret = result.err;
 210                         if (!ret) {
 211                                 reinit_completion(&result.completion);
 212                                 break;
 213                         }
 214                 }
 215                 /* fall through */
 216         default:
 217                 pr_err("Encryption of IV failed for GCM mode");
 218                 break;
 219         }
 220
 221         skcipher_request_free(sk_req);
 222         return ret;
 223 }
 224
 225 void omap_aes_gcm_dma_out_callback(void *data)
 226 {
 227         struct omap_aes_dev *dd = data;
 228         struct omap_aes_reqctx *rctx;
 229         int i, val;
 230         u32 *auth_tag, tag[4];
 231
 232         if (!(dd->flags & FLAGS_ENCRYPT))
 233                 scatterwalk_map_and_copy(tag, dd->aead_req->src,
 234                                          dd->total + dd->aead_req->assoclen,
 235                                          dd->authsize, 0);
 236
 237         rctx = aead_request_ctx(dd->aead_req);
 238         auth_tag = (u32 *)rctx->auth_tag;
 239         for (i = 0; i < 4; i++) {
 240                 val = omap_aes_read(dd, AES_REG_TAG_N(dd, i));
 241                 auth_tag[i] = val ^ auth_tag[i];
 242                 if (!(dd->flags & FLAGS_ENCRYPT))
 243                         auth_tag[i] = auth_tag[i] ^ tag[i];
 244         }
 245
 246         omap_aes_gcm_done_task(dd);
 247 }
 248
 249 static int omap_aes_gcm_handle_queue(struct omap_aes_dev *dd,
 250                                      struct aead_request *req)
 251 {
 252         struct omap_aes_ctx *ctx;
 253         struct aead_request *backlog;
 254         struct omap_aes_reqctx *rctx;
 255         unsigned long flags;
 256         int err, ret = 0;
 257
 258         spin_lock_irqsave(&dd->lock, flags);
 259         if (req)
 260                 ret = aead_enqueue_request(&dd->aead_queue, req);
 261         if (dd->flags & FLAGS_BUSY) {
 262                 spin_unlock_irqrestore(&dd->lock, flags);
 263                 return ret;
 264         }
 265
 266         backlog = aead_get_backlog(&dd->aead_queue);
 267         req = aead_dequeue_request(&dd->aead_queue);
 268         if (req)
 269                 dd->flags |= FLAGS_BUSY;
 270         spin_unlock_irqrestore(&dd->lock, flags);
 271
 272         if (!req)
 273                 return ret;
 274
 275         if (backlog)
 276                 backlog->base.complete(&backlog->base, -EINPROGRESS);
 277
 278         ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
 279         rctx = aead_request_ctx(req);
 280
 281         dd->ctx = ctx;
 282         rctx->dd = dd;
 283         dd->aead_req = req;
 284
 285         rctx->mode &= FLAGS_MODE_MASK;
 286         dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode;
 287
 288         err = omap_aes_gcm_copy_buffers(dd, req);
 289         if (err)
 290                 return err;
 291
 292         err = omap_aes_write_ctrl(dd);
 293         if (!err)
 294                 err = omap_aes_crypt_dma_start(dd);
 295
 296         if (err) {
 297                 omap_aes_gcm_finish_req(dd, err);
 298                 omap_aes_gcm_handle_queue(dd, NULL);
 299         }
 300
 301         return ret;
 302 }
 303
 304 static int omap_aes_gcm_crypt(struct aead_request *req, unsigned long mode)
 305 {
 306         struct omap_aes_reqctx *rctx = aead_request_ctx(req);
 307         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 308         unsigned int authlen = crypto_aead_authsize(aead);
 309         struct omap_aes_dev *dd;
 310         __be32 counter = cpu_to_be32(1);
 311         int err, assoclen;
 312
 313         memset(rctx->auth_tag, 0, sizeof(rctx->auth_tag));
 314         memcpy(rctx->iv + 12, &counter, 4);
 315
 316         err = do_encrypt_iv(req, (u32 *)rctx->auth_tag, (u32 *)rctx->iv);
 317         if (err)
 318                 return err;
 319
 320         if (mode & FLAGS_RFC4106_GCM)
 321                 assoclen = req->assoclen - 8;
 322         else
 323                 assoclen = req->assoclen;
 324         if (assoclen + req->cryptlen == 0) {
 325                 scatterwalk_map_and_copy(rctx->auth_tag, req->dst, 0, authlen,
 326                                          1);
 327                 return 0;
 328         }
 329
 330         dd = omap_aes_find_dev(rctx);
 331         if (!dd)
 332                 return -ENODEV;
 333         rctx->mode = mode;
 334
 335         return omap_aes_gcm_handle_queue(dd, req);
 336 }
 337
 338 int omap_aes_gcm_encrypt(struct aead_request *req)
 339 {
 340         struct omap_aes_reqctx *rctx = aead_request_ctx(req);
 341
 342         memcpy(rctx->iv, req->iv, 12);
 343         return omap_aes_gcm_crypt(req, FLAGS_ENCRYPT | FLAGS_GCM);
 344 }
 345
 346 int omap_aes_gcm_decrypt(struct aead_request *req)
 347 {
 348         struct omap_aes_reqctx *rctx = aead_request_ctx(req);
 349
 350         memcpy(rctx->iv, req->iv, 12);
 351         return omap_aes_gcm_crypt(req, FLAGS_GCM);
 352 }
 353
 354 int omap_aes_4106gcm_encrypt(struct aead_request *req)
 355 {
 356         struct omap_aes_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
 357         struct omap_aes_reqctx *rctx = aead_request_ctx(req);
 358
 359         memcpy(rctx->iv, ctx->nonce, 4);
 360         memcpy(rctx->iv + 4, req->iv, 8);
 361         return omap_aes_gcm_crypt(req, FLAGS_ENCRYPT | FLAGS_GCM |
 362                                   FLAGS_RFC4106_GCM);
 363 }
 364
 365 int omap_aes_4106gcm_decrypt(struct aead_request *req)
 366 {
 367         struct omap_aes_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
 368         struct omap_aes_reqctx *rctx = aead_request_ctx(req);
 369
 370         memcpy(rctx->iv, ctx->nonce, 4);
 371         memcpy(rctx->iv + 4, req->iv, 8);
 372         return omap_aes_gcm_crypt(req, FLAGS_GCM | FLAGS_RFC4106_GCM);
 373 }
 374
 375 int omap_aes_gcm_setkey(struct crypto_aead *tfm, const u8 *key,
 376                         unsigned int keylen)
 377 {
 378         struct omap_aes_ctx *ctx = crypto_aead_ctx(tfm);
 379
 380         if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
 381             keylen != AES_KEYSIZE_256)
 382                 return -EINVAL;
 383
 384         memcpy(ctx->key, key, keylen);
 385         ctx->keylen = keylen;
 386
 387         return 0;
 388 }
 389
 390 int omap_aes_4106gcm_setkey(struct crypto_aead *tfm, const u8 *key,
 391                             unsigned int keylen)
 392 {
 393         struct omap_aes_ctx *ctx = crypto_aead_ctx(tfm);
 394
 395         if (keylen < 4)
 396                 return -EINVAL;
 397
 398         keylen -= 4;
 399         if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
 400             keylen != AES_KEYSIZE_256)
 401                 return -EINVAL;
 402
 403         memcpy(ctx->key, key, keylen);
 404         memcpy(ctx->nonce, key + keylen, 4);
 405         ctx->keylen = keylen;
 406
 407         return 0;
 408 }