crypto/chacha20poly1305.c

   1 /*
   2  * ChaCha20-Poly1305 AEAD, RFC7539
   3  *
   4  * Copyright (C) 2015 Martin Willi
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License as published by
   8  * the Free Software Foundation; either version 2 of the License, or
   9  * (at your option) any later version.
  10  */
  11
  12 #include <crypto/internal/aead.h>
  13 #include <crypto/internal/hash.h>
  14 #include <crypto/internal/skcipher.h>
  15 #include <crypto/scatterwalk.h>
  16 #include <crypto/chacha20.h>
  17 #include <crypto/poly1305.h>
  18 #include <linux/err.h>
  19 #include <linux/init.h>
  20 #include <linux/kernel.h>
  21 #include <linux/module.h>
  22
  23 #include "internal.h"
  24
  25 #define CHACHAPOLY_IV_SIZE      12
  26
  27 struct chachapoly_instance_ctx {
  28         struct crypto_skcipher_spawn chacha;
  29         struct crypto_ahash_spawn poly;
  30         unsigned int saltlen;
  31 };
  32
  33 struct chachapoly_ctx {
  34         struct crypto_ablkcipher *chacha;
  35         struct crypto_ahash *poly;
  36         /* key bytes we use for the ChaCha20 IV */
  37         unsigned int saltlen;
  38         u8 salt[];
  39 };
  40
  41 struct poly_req {
  42         /* zero byte padding for AD/ciphertext, as needed */
  43         u8 pad[POLY1305_BLOCK_SIZE];
  44         /* tail data with AD/ciphertext lengths */
  45         struct {
  46                 __le64 assoclen;
  47                 __le64 cryptlen;
  48         } tail;
  49         struct scatterlist src[1];
  50         struct ahash_request req; /* must be last member */
  51 };
  52
  53 struct chacha_req {
  54         u8 iv[CHACHA20_IV_SIZE];
  55         struct scatterlist src[1];
  56         struct ablkcipher_request req; /* must be last member */
  57 };
  58
  59 struct chachapoly_req_ctx {
  60         struct scatterlist src[2];
  61         struct scatterlist dst[2];
  62         /* the key we generate for Poly1305 using Chacha20 */
  63         u8 key[POLY1305_KEY_SIZE];
  64         /* calculated Poly1305 tag */
  65         u8 tag[POLY1305_DIGEST_SIZE];
  66         /* length of data to en/decrypt, without ICV */
  67         unsigned int cryptlen;
  68         /* Actual AD, excluding IV */
  69         unsigned int assoclen;
  70         union {
  71                 struct poly_req poly;
  72                 struct chacha_req chacha;
  73         } u;
  74 };
  75
  76 static inline void async_done_continue(struct aead_request *req, int err,
  77                                        int (*cont)(struct aead_request *))
  78 {
  79         if (!err)
  80                 err = cont(req);
  81
  82         if (err != -EINPROGRESS && err != -EBUSY)
  83                 aead_request_complete(req, err);
  84 }
  85
  86 static void chacha_iv(u8 *iv, struct aead_request *req, u32 icb)
  87 {
  88         struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
  89         __le32 leicb = cpu_to_le32(icb);
  90
  91         memcpy(iv, &leicb, sizeof(leicb));
  92         memcpy(iv + sizeof(leicb), ctx->salt, ctx->saltlen);
  93         memcpy(iv + sizeof(leicb) + ctx->saltlen, req->iv,
  94                CHACHA20_IV_SIZE - sizeof(leicb) - ctx->saltlen);
  95 }
  96
  97 static int poly_verify_tag(struct aead_request *req)
  98 {
  99         struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
 100         u8 tag[sizeof(rctx->tag)];
 101
 102         scatterwalk_map_and_copy(tag, req->src,
 103                                  req->assoclen + rctx->cryptlen,
 104                                  sizeof(tag), 0);
 105         if (crypto_memneq(tag, rctx->tag, sizeof(tag)))
 106                 return -EBADMSG;
 107         return 0;
 108 }
 109
 110 static int poly_copy_tag(struct aead_request *req)
 111 {
 112         struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
 113
 114         scatterwalk_map_and_copy(rctx->tag, req->dst,
 115                                  req->assoclen + rctx->cryptlen,
 116                                  sizeof(rctx->tag), 1);
 117         return 0;
 118 }
 119
 120 static void chacha_decrypt_done(struct crypto_async_request *areq, int err)
 121 {
 122         async_done_continue(areq->data, err, poly_verify_tag);
 123 }
 124
 125 static int chacha_decrypt(struct aead_request *req)
 126 {
 127         struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
 128         struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
 129         struct chacha_req *creq = &rctx->u.chacha;
 130         struct scatterlist *src, *dst;
 131         int err;
 132
 133         if (rctx->cryptlen == 0)
 134                 goto skip;
 135
 136         chacha_iv(creq->iv, req, 1);
 137
 138         sg_init_table(rctx->src, 2);
 139         src = scatterwalk_ffwd(rctx->src, req->src, req->assoclen);
 140         dst = src;
 141
 142         if (req->src != req->dst) {
 143                 sg_init_table(rctx->dst, 2);
 144                 dst = scatterwalk_ffwd(rctx->dst, req->dst, req->assoclen);
 145         }
 146
 147         ablkcipher_request_set_callback(&creq->req, aead_request_flags(req),
 148                                         chacha_decrypt_done, req);
 149         ablkcipher_request_set_tfm(&creq->req, ctx->chacha);
 150         ablkcipher_request_set_crypt(&creq->req, src, dst,
 151                                      rctx->cryptlen, creq->iv);
 152         err = crypto_ablkcipher_decrypt(&creq->req);
 153         if (err)
 154                 return err;
 155
 156 skip:
 157         return poly_verify_tag(req);
 158 }
 159
 160 static int poly_tail_continue(struct aead_request *req)
 161 {
 162         struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
 163
 164         if (rctx->cryptlen == req->cryptlen) /* encrypting */
 165                 return poly_copy_tag(req);
 166
 167         return chacha_decrypt(req);
 168 }
 169
 170 static void poly_tail_done(struct crypto_async_request *areq, int err)
 171 {
 172         async_done_continue(areq->data, err, poly_tail_continue);
 173 }
 174
 175 static int poly_tail(struct aead_request *req)
 176 {
 177         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 178         struct chachapoly_ctx *ctx = crypto_aead_ctx(tfm);
 179         struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
 180         struct poly_req *preq = &rctx->u.poly;
 181         __le64 len;
 182         int err;
 183
 184         sg_init_table(preq->src, 1);
 185         len = cpu_to_le64(rctx->assoclen);
 186         memcpy(&preq->tail.assoclen, &len, sizeof(len));
 187         len = cpu_to_le64(rctx->cryptlen);
 188         memcpy(&preq->tail.cryptlen, &len, sizeof(len));
 189         sg_set_buf(preq->src, &preq->tail, sizeof(preq->tail));
 190
 191         ahash_request_set_callback(&preq->req, aead_request_flags(req),
 192                                    poly_tail_done, req);
 193         ahash_request_set_tfm(&preq->req, ctx->poly);
 194         ahash_request_set_crypt(&preq->req, preq->src,
 195                                 rctx->tag, sizeof(preq->tail));
 196
 197         err = crypto_ahash_finup(&preq->req);
 198         if (err)
 199                 return err;
 200
 201         return poly_tail_continue(req);
 202 }
 203
 204 static void poly_cipherpad_done(struct crypto_async_request *areq, int err)
 205 {
 206         async_done_continue(areq->data, err, poly_tail);
 207 }
 208
 209 static int poly_cipherpad(struct aead_request *req)
 210 {
 211         struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
 212         struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
 213         struct poly_req *preq = &rctx->u.poly;
 214         unsigned int padlen, bs = POLY1305_BLOCK_SIZE;
 215         int err;
 216
 217         padlen = (bs - (rctx->cryptlen % bs)) % bs;
 218         memset(preq->pad, 0, sizeof(preq->pad));
 219         sg_init_table(preq->src, 1);
 220         sg_set_buf(preq->src, &preq->pad, padlen);
 221
 222         ahash_request_set_callback(&preq->req, aead_request_flags(req),
 223                                    poly_cipherpad_done, req);
 224         ahash_request_set_tfm(&preq->req, ctx->poly);
 225         ahash_request_set_crypt(&preq->req, preq->src, NULL, padlen);
 226
 227         err = crypto_ahash_update(&preq->req);
 228         if (err)
 229                 return err;
 230
 231         return poly_tail(req);
 232 }
 233
 234 static void poly_cipher_done(struct crypto_async_request *areq, int err)
 235 {
 236         async_done_continue(areq->data, err, poly_cipherpad);
 237 }
 238
 239 static int poly_cipher(struct aead_request *req)
 240 {
 241         struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
 242         struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
 243         struct poly_req *preq = &rctx->u.poly;
 244         struct scatterlist *crypt = req->src;
 245         int err;
 246
 247         if (rctx->cryptlen == req->cryptlen) /* encrypting */
 248                 crypt = req->dst;
 249
 250         sg_init_table(rctx->src, 2);
 251         crypt = scatterwalk_ffwd(rctx->src, crypt, req->assoclen);
 252
 253         ahash_request_set_callback(&preq->req, aead_request_flags(req),
 254                                    poly_cipher_done, req);
 255         ahash_request_set_tfm(&preq->req, ctx->poly);
 256         ahash_request_set_crypt(&preq->req, crypt, NULL, rctx->cryptlen);
 257
 258         err = crypto_ahash_update(&preq->req);
 259         if (err)
 260                 return err;
 261
 262         return poly_cipherpad(req);
 263 }
 264
 265 static void poly_adpad_done(struct crypto_async_request *areq, int err)
 266 {
 267         async_done_continue(areq->data, err, poly_cipher);
 268 }
 269
 270 static int poly_adpad(struct aead_request *req)
 271 {
 272         struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
 273         struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
 274         struct poly_req *preq = &rctx->u.poly;
 275         unsigned int padlen, bs = POLY1305_BLOCK_SIZE;
 276         int err;
 277
 278         padlen = (bs - (rctx->assoclen % bs)) % bs;
 279         memset(preq->pad, 0, sizeof(preq->pad));
 280         sg_init_table(preq->src, 1);
 281         sg_set_buf(preq->src, preq->pad, padlen);
 282
 283         ahash_request_set_callback(&preq->req, aead_request_flags(req),
 284                                    poly_adpad_done, req);
 285         ahash_request_set_tfm(&preq->req, ctx->poly);
 286         ahash_request_set_crypt(&preq->req, preq->src, NULL, padlen);
 287
 288         err = crypto_ahash_update(&preq->req);
 289         if (err)
 290                 return err;
 291
 292         return poly_cipher(req);
 293 }
 294
 295 static void poly_ad_done(struct crypto_async_request *areq, int err)
 296 {
 297         async_done_continue(areq->data, err, poly_adpad);
 298 }
 299
 300 static int poly_ad(struct aead_request *req)
 301 {
 302         struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
 303         struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
 304         struct poly_req *preq = &rctx->u.poly;
 305         int err;
 306
 307         ahash_request_set_callback(&preq->req, aead_request_flags(req),
 308                                    poly_ad_done, req);
 309         ahash_request_set_tfm(&preq->req, ctx->poly);
 310         ahash_request_set_crypt(&preq->req, req->src, NULL, rctx->assoclen);
 311
 312         err = crypto_ahash_update(&preq->req);
 313         if (err)
 314                 return err;
 315
 316         return poly_adpad(req);
 317 }
 318
 319 static void poly_setkey_done(struct crypto_async_request *areq, int err)
 320 {
 321         async_done_continue(areq->data, err, poly_ad);
 322 }
 323
 324 static int poly_setkey(struct aead_request *req)
 325 {
 326         struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
 327         struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
 328         struct poly_req *preq = &rctx->u.poly;
 329         int err;
 330
 331         sg_init_table(preq->src, 1);
 332         sg_set_buf(preq->src, rctx->key, sizeof(rctx->key));
 333
 334         ahash_request_set_callback(&preq->req, aead_request_flags(req),
 335                                    poly_setkey_done, req);
 336         ahash_request_set_tfm(&preq->req, ctx->poly);
 337         ahash_request_set_crypt(&preq->req, preq->src, NULL, sizeof(rctx->key));
 338
 339         err = crypto_ahash_update(&preq->req);
 340         if (err)
 341                 return err;
 342
 343         return poly_ad(req);
 344 }
 345
 346 static void poly_init_done(struct crypto_async_request *areq, int err)
 347 {
 348         async_done_continue(areq->data, err, poly_setkey);
 349 }
 350
 351 static int poly_init(struct aead_request *req)
 352 {
 353         struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
 354         struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
 355         struct poly_req *preq = &rctx->u.poly;
 356         int err;
 357
 358         ahash_request_set_callback(&preq->req, aead_request_flags(req),
 359                                    poly_init_done, req);
 360         ahash_request_set_tfm(&preq->req, ctx->poly);
 361
 362         err = crypto_ahash_init(&preq->req);
 363         if (err)
 364                 return err;
 365
 366         return poly_setkey(req);
 367 }
 368
 369 static void poly_genkey_done(struct crypto_async_request *areq, int err)
 370 {
 371         async_done_continue(areq->data, err, poly_init);
 372 }
 373
 374 static int poly_genkey(struct aead_request *req)
 375 {
 376         struct crypto_aead *tfm = crypto_aead_reqtfm(req);
 377         struct chachapoly_ctx *ctx = crypto_aead_ctx(tfm);
 378         struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
 379         struct chacha_req *creq = &rctx->u.chacha;
 380         int err;
 381
 382         rctx->assoclen = req->assoclen;
 383
 384         if (crypto_aead_ivsize(tfm) == 8) {
 385                 if (rctx->assoclen < 8)
 386                         return -EINVAL;
 387                 rctx->assoclen -= 8;
 388         }
 389
 390         sg_init_table(creq->src, 1);
 391         memset(rctx->key, 0, sizeof(rctx->key));
 392         sg_set_buf(creq->src, rctx->key, sizeof(rctx->key));
 393
 394         chacha_iv(creq->iv, req, 0);
 395
 396         ablkcipher_request_set_callback(&creq->req, aead_request_flags(req),
 397                                         poly_genkey_done, req);
 398         ablkcipher_request_set_tfm(&creq->req, ctx->chacha);
 399         ablkcipher_request_set_crypt(&creq->req, creq->src, creq->src,
 400                                      POLY1305_KEY_SIZE, creq->iv);
 401
 402         err = crypto_ablkcipher_decrypt(&creq->req);
 403         if (err)
 404                 return err;
 405
 406         return poly_init(req);
 407 }
 408
 409 static void chacha_encrypt_done(struct crypto_async_request *areq, int err)
 410 {
 411         async_done_continue(areq->data, err, poly_genkey);
 412 }
 413
 414 static int chacha_encrypt(struct aead_request *req)
 415 {
 416         struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req));
 417         struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
 418         struct chacha_req *creq = &rctx->u.chacha;
 419         struct scatterlist *src, *dst;
 420         int err;
 421
 422         if (req->cryptlen == 0)
 423                 goto skip;
 424
 425         chacha_iv(creq->iv, req, 1);
 426
 427         sg_init_table(rctx->src, 2);
 428         src = scatterwalk_ffwd(rctx->src, req->src, req->assoclen);
 429         dst = src;
 430
 431         if (req->src != req->dst) {
 432                 sg_init_table(rctx->dst, 2);
 433                 dst = scatterwalk_ffwd(rctx->dst, req->dst, req->assoclen);
 434         }
 435
 436         ablkcipher_request_set_callback(&creq->req, aead_request_flags(req),
 437                                         chacha_encrypt_done, req);
 438         ablkcipher_request_set_tfm(&creq->req, ctx->chacha);
 439         ablkcipher_request_set_crypt(&creq->req, src, dst,
 440                                      req->cryptlen, creq->iv);
 441         err = crypto_ablkcipher_encrypt(&creq->req);
 442         if (err)
 443                 return err;
 444
 445 skip:
 446         return poly_genkey(req);
 447 }
 448
 449 static int chachapoly_encrypt(struct aead_request *req)
 450 {
 451         struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
 452
 453         rctx->cryptlen = req->cryptlen;
 454
 455         /* encrypt call chain:
 456          * - chacha_encrypt/done()
 457          * - poly_genkey/done()
 458          * - poly_init/done()
 459          * - poly_setkey/done()
 460          * - poly_ad/done()
 461          * - poly_adpad/done()
 462          * - poly_cipher/done()
 463          * - poly_cipherpad/done()
 464          * - poly_tail/done/continue()
 465          * - poly_copy_tag()
 466          */
 467         return chacha_encrypt(req);
 468 }
 469
 470 static int chachapoly_decrypt(struct aead_request *req)
 471 {
 472         struct chachapoly_req_ctx *rctx = aead_request_ctx(req);
 473
 474         rctx->cryptlen = req->cryptlen - POLY1305_DIGEST_SIZE;
 475
 476         /* decrypt call chain:
 477          * - poly_genkey/done()
 478          * - poly_init/done()
 479          * - poly_setkey/done()
 480          * - poly_ad/done()
 481          * - poly_adpad/done()
 482          * - poly_cipher/done()
 483          * - poly_cipherpad/done()
 484          * - poly_tail/done/continue()
 485          * - chacha_decrypt/done()
 486          * - poly_verify_tag()
 487          */
 488         return poly_genkey(req);
 489 }
 490
 491 static int chachapoly_setkey(struct crypto_aead *aead, const u8 *key,
 492                              unsigned int keylen)
 493 {
 494         struct chachapoly_ctx *ctx = crypto_aead_ctx(aead);
 495         int err;
 496
 497         if (keylen != ctx->saltlen + CHACHA20_KEY_SIZE)
 498                 return -EINVAL;
 499
 500         keylen -= ctx->saltlen;
 501         memcpy(ctx->salt, key + keylen, ctx->saltlen);
 502
 503         crypto_ablkcipher_clear_flags(ctx->chacha, CRYPTO_TFM_REQ_MASK);
 504         crypto_ablkcipher_set_flags(ctx->chacha, crypto_aead_get_flags(aead) &
 505                                     CRYPTO_TFM_REQ_MASK);
 506
 507         err = crypto_ablkcipher_setkey(ctx->chacha, key, keylen);
 508         crypto_aead_set_flags(aead, crypto_ablkcipher_get_flags(ctx->chacha) &
 509                               CRYPTO_TFM_RES_MASK);
 510         return err;
 511 }
 512
 513 static int chachapoly_setauthsize(struct crypto_aead *tfm,
 514                                   unsigned int authsize)
 515 {
 516         if (authsize != POLY1305_DIGEST_SIZE)
 517                 return -EINVAL;
 518
 519         return 0;
 520 }
 521
 522 static int chachapoly_init(struct crypto_aead *tfm)
 523 {
 524         struct aead_instance *inst = aead_alg_instance(tfm);
 525         struct chachapoly_instance_ctx *ictx = aead_instance_ctx(inst);
 526         struct chachapoly_ctx *ctx = crypto_aead_ctx(tfm);
 527         struct crypto_ablkcipher *chacha;
 528         struct crypto_ahash *poly;
 529         unsigned long align;
 530
 531         poly = crypto_spawn_ahash(&ictx->poly);
 532         if (IS_ERR(poly))
 533                 return PTR_ERR(poly);
 534
 535         chacha = crypto_spawn_skcipher(&ictx->chacha);
 536         if (IS_ERR(chacha)) {
 537                 crypto_free_ahash(poly);
 538                 return PTR_ERR(chacha);
 539         }
 540
 541         ctx->chacha = chacha;
 542         ctx->poly = poly;
 543         ctx->saltlen = ictx->saltlen;
 544
 545         align = crypto_aead_alignmask(tfm);
 546         align &= ~(crypto_tfm_ctx_alignment() - 1);
 547         crypto_aead_set_reqsize(
 548                 tfm,
 549                 align + offsetof(struct chachapoly_req_ctx, u) +
 550                 max(offsetof(struct chacha_req, req) +
 551                     sizeof(struct ablkcipher_request) +
 552                     crypto_ablkcipher_reqsize(chacha),
 553                     offsetof(struct poly_req, req) +
 554                     sizeof(struct ahash_request) +
 555                     crypto_ahash_reqsize(poly)));
 556
 557         return 0;
 558 }
 559
 560 static void chachapoly_exit(struct crypto_aead *tfm)
 561 {
 562         struct chachapoly_ctx *ctx = crypto_aead_ctx(tfm);
 563
 564         crypto_free_ahash(ctx->poly);
 565         crypto_free_ablkcipher(ctx->chacha);
 566 }
 567
 568 static void chachapoly_free(struct aead_instance *inst)
 569 {
 570         struct chachapoly_instance_ctx *ctx = aead_instance_ctx(inst);
 571
 572         crypto_drop_skcipher(&ctx->chacha);
 573         crypto_drop_ahash(&ctx->poly);
 574         kfree(inst);
 575 }
 576
 577 static int chachapoly_create(struct crypto_template *tmpl, struct rtattr **tb,
 578                              const char *name, unsigned int ivsize)
 579 {
 580         struct crypto_attr_type *algt;
 581         struct aead_instance *inst;
 582         struct crypto_alg *chacha;
 583         struct crypto_alg *poly;
 584         struct hash_alg_common *poly_hash;
 585         struct chachapoly_instance_ctx *ctx;
 586         const char *chacha_name, *poly_name;
 587         int err;
 588
 589         if (ivsize > CHACHAPOLY_IV_SIZE)
 590                 return -EINVAL;
 591
 592         algt = crypto_get_attr_type(tb);
 593         if (IS_ERR(algt))
 594                 return PTR_ERR(algt);
 595
 596         if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask)
 597                 return -EINVAL;
 598
 599         chacha_name = crypto_attr_alg_name(tb[1]);
 600         if (IS_ERR(chacha_name))
 601                 return PTR_ERR(chacha_name);
 602         poly_name = crypto_attr_alg_name(tb[2]);
 603         if (IS_ERR(poly_name))
 604                 return PTR_ERR(poly_name);
 605
 606         poly = crypto_find_alg(poly_name, &crypto_ahash_type,
 607                                CRYPTO_ALG_TYPE_HASH,
 608                                CRYPTO_ALG_TYPE_AHASH_MASK);
 609         if (IS_ERR(poly))
 610                 return PTR_ERR(poly);
 611
 612         err = -ENOMEM;
 613         inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
 614         if (!inst)
 615                 goto out_put_poly;
 616
 617         ctx = aead_instance_ctx(inst);
 618         ctx->saltlen = CHACHAPOLY_IV_SIZE - ivsize;
 619         poly_hash = __crypto_hash_alg_common(poly);
 620         err = crypto_init_ahash_spawn(&ctx->poly, poly_hash,
 621                                       aead_crypto_instance(inst));
 622         if (err)
 623                 goto err_free_inst;
 624
 625         crypto_set_skcipher_spawn(&ctx->chacha, aead_crypto_instance(inst));
 626         err = crypto_grab_skcipher(&ctx->chacha, chacha_name, 0,
 627                                    crypto_requires_sync(algt->type,
 628                                                         algt->mask));
 629         if (err)
 630                 goto err_drop_poly;
 631
 632         chacha = crypto_skcipher_spawn_alg(&ctx->chacha);
 633
 634         err = -EINVAL;
 635         /* Need 16-byte IV size, including Initial Block Counter value */
 636         if (chacha->cra_ablkcipher.ivsize != CHACHA20_IV_SIZE)
 637                 goto out_drop_chacha;
 638         /* Not a stream cipher? */
 639         if (chacha->cra_blocksize != 1)
 640                 goto out_drop_chacha;
 641
 642         err = -ENAMETOOLONG;
 643         if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME,
 644                      "%s(%s,%s)", name, chacha_name,
 645                      poly_name) >= CRYPTO_MAX_ALG_NAME)
 646                 goto out_drop_chacha;
 647         if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME,
 648                      "%s(%s,%s)", name, chacha->cra_driver_name,
 649                      poly->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
 650                 goto out_drop_chacha;
 651
 652         inst->alg.base.cra_flags = (chacha->cra_flags | poly->cra_flags) &
 653                                    CRYPTO_ALG_ASYNC;
 654         inst->alg.base.cra_priority = (chacha->cra_priority +
 655                                        poly->cra_priority) / 2;
 656         inst->alg.base.cra_blocksize = 1;
 657         inst->alg.base.cra_alignmask = chacha->cra_alignmask |
 658                                        poly->cra_alignmask;
 659         inst->alg.base.cra_ctxsize = sizeof(struct chachapoly_ctx) +
 660                                      ctx->saltlen;
 661         inst->alg.ivsize = ivsize;
 662         inst->alg.maxauthsize = POLY1305_DIGEST_SIZE;
 663         inst->alg.init = chachapoly_init;
 664         inst->alg.exit = chachapoly_exit;
 665         inst->alg.encrypt = chachapoly_encrypt;
 666         inst->alg.decrypt = chachapoly_decrypt;
 667         inst->alg.setkey = chachapoly_setkey;
 668         inst->alg.setauthsize = chachapoly_setauthsize;
 669
 670         inst->free = chachapoly_free;
 671
 672         err = aead_register_instance(tmpl, inst);
 673         if (err)
 674                 goto out_drop_chacha;
 675
 676 out_put_poly:
 677         crypto_mod_put(poly);
 678         return err;
 679
 680 out_drop_chacha:
 681         crypto_drop_skcipher(&ctx->chacha);
 682 err_drop_poly:
 683         crypto_drop_ahash(&ctx->poly);
 684 err_free_inst:
 685         kfree(inst);
 686         goto out_put_poly;
 687 }
 688
 689 static int rfc7539_create(struct crypto_template *tmpl, struct rtattr **tb)
 690 {
 691         return chachapoly_create(tmpl, tb, "rfc7539", 12);
 692 }
 693
 694 static int rfc7539esp_create(struct crypto_template *tmpl, struct rtattr **tb)
 695 {
 696         return chachapoly_create(tmpl, tb, "rfc7539esp", 8);
 697 }
 698
 699 static struct crypto_template rfc7539_tmpl = {
 700         .name = "rfc7539",
 701         .create = rfc7539_create,
 702         .module = THIS_MODULE,
 703 };
 704
 705 static struct crypto_template rfc7539esp_tmpl = {
 706         .name = "rfc7539esp",
 707         .create = rfc7539esp_create,
 708         .module = THIS_MODULE,
 709 };
 710
 711 static int __init chacha20poly1305_module_init(void)
 712 {
 713         int err;
 714
 715         err = crypto_register_template(&rfc7539_tmpl);
 716         if (err)
 717                 return err;
 718
 719         err = crypto_register_template(&rfc7539esp_tmpl);
 720         if (err)
 721                 crypto_unregister_template(&rfc7539_tmpl);
 722
 723         return err;
 724 }
 725
 726 static void __exit chacha20poly1305_module_exit(void)
 727 {
 728         crypto_unregister_template(&rfc7539esp_tmpl);
 729         crypto_unregister_template(&rfc7539_tmpl);
 730 }
 731
 732 module_init(chacha20poly1305_module_init);
 733 module_exit(chacha20poly1305_module_exit);
 734
 735 MODULE_LICENSE("GPL");
 736 MODULE_AUTHOR("Martin Willi <martin@strongswan.org>");
 737 MODULE_DESCRIPTION("ChaCha20-Poly1305 AEAD");
 738 MODULE_ALIAS_CRYPTO("rfc7539");
 739 MODULE_ALIAS_CRYPTO("rfc7539esp");