crypto/algif_skcipher.c

   1 /*
   2  * algif_skcipher: User-space interface for skcipher algorithms
   3  *
   4  * This file provides the user-space API for symmetric key ciphers.
   5  *
   6  * Copyright (c) 2010 Herbert Xu <herbert@gondor.apana.org.au>
   7  *
   8  * This program is free software; you can redistribute it and/or modify it
   9  * under the terms of the GNU General Public License as published by the Free
  10  * Software Foundation; either version 2 of the License, or (at your option)
  11  * any later version.
  12  *
  13  */
  14
  15 #include <crypto/scatterwalk.h>
  16 #include <crypto/skcipher.h>
  17 #include <crypto/if_alg.h>
  18 #include <linux/init.h>
  19 #include <linux/list.h>
  20 #include <linux/kernel.h>
  21 #include <linux/sched/signal.h>
  22 #include <linux/mm.h>
  23 #include <linux/module.h>
  24 #include <linux/net.h>
  25 #include <net/sock.h>
  26
  27 struct skcipher_sg_list {
  28         struct list_head list;
  29
  30         int cur;
  31
  32         struct scatterlist sg[0];
  33 };
  34
  35 struct skcipher_tfm {
  36         struct crypto_skcipher *skcipher;
  37         bool has_key;
  38 };
  39
  40 struct skcipher_ctx {
  41         struct list_head tsgl;
  42         struct af_alg_sgl rsgl;
  43
  44         void *iv;
  45
  46         struct af_alg_completion completion;
  47
  48         atomic_t inflight;
  49         size_t used;
  50
  51         unsigned int len;
  52         bool more;
  53         bool merge;
  54         bool enc;
  55
  56         struct skcipher_request req;
  57 };
  58
  59 struct skcipher_async_rsgl {
  60         struct af_alg_sgl sgl;
  61         struct list_head list;
  62 };
  63
  64 struct skcipher_async_req {
  65         struct kiocb *iocb;
  66         struct skcipher_async_rsgl first_sgl;
  67         struct list_head list;
  68         struct scatterlist *tsg;
  69         atomic_t *inflight;
  70         struct skcipher_request req;
  71 };
  72
  73 #define MAX_SGL_ENTS ((4096 - sizeof(struct skcipher_sg_list)) / \
  74                       sizeof(struct scatterlist) - 1)
  75
  76 static void skcipher_free_async_sgls(struct skcipher_async_req *sreq)
  77 {
  78         struct skcipher_async_rsgl *rsgl, *tmp;
  79         struct scatterlist *sgl;
  80         struct scatterlist *sg;
  81         int i, n;
  82
  83         list_for_each_entry_safe(rsgl, tmp, &sreq->list, list) {
  84                 af_alg_free_sg(&rsgl->sgl);
  85                 if (rsgl != &sreq->first_sgl)
  86                         kfree(rsgl);
  87         }
  88         sgl = sreq->tsg;
  89         n = sg_nents(sgl);
  90         for_each_sg(sgl, sg, n, i) {
  91                 struct page *page = sg_page(sg);
  92
  93                 /* some SGs may not have a page mapped */
  94                 if (page && page_ref_count(page))
  95                         put_page(page);
  96         }
  97
  98         kfree(sreq->tsg);
  99 }
 100
 101 static void skcipher_async_cb(struct crypto_async_request *req, int err)
 102 {
 103         struct skcipher_async_req *sreq = req->data;
 104         struct kiocb *iocb = sreq->iocb;
 105
 106         atomic_dec(sreq->inflight);
 107         skcipher_free_async_sgls(sreq);
 108         kzfree(sreq);
 109         iocb->ki_complete(iocb, err, err);
 110 }
 111
 112 static inline int skcipher_sndbuf(struct sock *sk)
 113 {
 114         struct alg_sock *ask = alg_sk(sk);
 115         struct skcipher_ctx *ctx = ask->private;
 116
 117         return max_t(int, max_t(int, sk->sk_sndbuf & PAGE_MASK, PAGE_SIZE) -
 118                           ctx->used, 0);
 119 }
 120
 121 static inline bool skcipher_writable(struct sock *sk)
 122 {
 123         return PAGE_SIZE <= skcipher_sndbuf(sk);
 124 }
 125
 126 static int skcipher_alloc_sgl(struct sock *sk)
 127 {
 128         struct alg_sock *ask = alg_sk(sk);
 129         struct skcipher_ctx *ctx = ask->private;
 130         struct skcipher_sg_list *sgl;
 131         struct scatterlist *sg = NULL;
 132
 133         sgl = list_entry(ctx->tsgl.prev, struct skcipher_sg_list, list);
 134         if (!list_empty(&ctx->tsgl))
 135                 sg = sgl->sg;
 136
 137         if (!sg || sgl->cur >= MAX_SGL_ENTS) {
 138                 sgl = sock_kmalloc(sk, sizeof(*sgl) +
 139                                        sizeof(sgl->sg[0]) * (MAX_SGL_ENTS + 1),
 140                                    GFP_KERNEL);
 141                 if (!sgl)
 142                         return -ENOMEM;
 143
 144                 sg_init_table(sgl->sg, MAX_SGL_ENTS + 1);
 145                 sgl->cur = 0;
 146
 147                 if (sg) {
 148                         sg_chain(sg, MAX_SGL_ENTS + 1, sgl->sg);
 149                         sg_unmark_end(sg + (MAX_SGL_ENTS - 1));
 150                 }
 151
 152                 list_add_tail(&sgl->list, &ctx->tsgl);
 153         }
 154
 155         return 0;
 156 }
 157
 158 static void skcipher_pull_sgl(struct sock *sk, size_t used, int put)
 159 {
 160         struct alg_sock *ask = alg_sk(sk);
 161         struct skcipher_ctx *ctx = ask->private;
 162         struct skcipher_sg_list *sgl;
 163         struct scatterlist *sg;
 164         int i;
 165
 166         while (!list_empty(&ctx->tsgl)) {
 167                 sgl = list_first_entry(&ctx->tsgl, struct skcipher_sg_list,
 168                                        list);
 169                 sg = sgl->sg;
 170
 171                 for (i = 0; i < sgl->cur; i++) {
 172                         size_t plen = min_t(size_t, used, sg[i].length);
 173
 174                         if (!sg_page(sg + i))
 175                                 continue;
 176
 177                         sg[i].length -= plen;
 178                         sg[i].offset += plen;
 179
 180                         used -= plen;
 181                         ctx->used -= plen;
 182
 183                         if (sg[i].length)
 184                                 return;
 185                         if (put)
 186                                 put_page(sg_page(sg + i));
 187                         sg_assign_page(sg + i, NULL);
 188                 }
 189
 190                 list_del(&sgl->list);
 191                 sock_kfree_s(sk, sgl,
 192                              sizeof(*sgl) + sizeof(sgl->sg[0]) *
 193                                             (MAX_SGL_ENTS + 1));
 194         }
 195
 196         if (!ctx->used)
 197                 ctx->merge = 0;
 198 }
 199
 200 static void skcipher_free_sgl(struct sock *sk)
 201 {
 202         struct alg_sock *ask = alg_sk(sk);
 203         struct skcipher_ctx *ctx = ask->private;
 204
 205         skcipher_pull_sgl(sk, ctx->used, 1);
 206 }
 207
 208 static int skcipher_wait_for_wmem(struct sock *sk, unsigned flags)
 209 {
 210         DEFINE_WAIT_FUNC(wait, woken_wake_function);
 211         int err = -ERESTARTSYS;
 212         long timeout;
 213
 214         if (flags & MSG_DONTWAIT)
 215                 return -EAGAIN;
 216
 217         sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
 218
 219         add_wait_queue(sk_sleep(sk), &wait);
 220         for (;;) {
 221                 if (signal_pending(current))
 222                         break;
 223                 timeout = MAX_SCHEDULE_TIMEOUT;
 224                 if (sk_wait_event(sk, &timeout, skcipher_writable(sk), &wait)) {
 225                         err = 0;
 226                         break;
 227                 }
 228         }
 229         remove_wait_queue(sk_sleep(sk), &wait);
 230
 231         return err;
 232 }
 233
 234 static void skcipher_wmem_wakeup(struct sock *sk)
 235 {
 236         struct socket_wq *wq;
 237
 238         if (!skcipher_writable(sk))
 239                 return;
 240
 241         rcu_read_lock();
 242         wq = rcu_dereference(sk->sk_wq);
 243         if (skwq_has_sleeper(wq))
 244                 wake_up_interruptible_sync_poll(&wq->wait, POLLIN |
 245                                                            POLLRDNORM |
 246                                                            POLLRDBAND);
 247         sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
 248         rcu_read_unlock();
 249 }
 250
 251 static int skcipher_wait_for_data(struct sock *sk, unsigned flags)
 252 {
 253         DEFINE_WAIT_FUNC(wait, woken_wake_function);
 254         struct alg_sock *ask = alg_sk(sk);
 255         struct skcipher_ctx *ctx = ask->private;
 256         long timeout;
 257         int err = -ERESTARTSYS;
 258
 259         if (flags & MSG_DONTWAIT) {
 260                 return -EAGAIN;
 261         }
 262
 263         sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
 264
 265         add_wait_queue(sk_sleep(sk), &wait);
 266         for (;;) {
 267                 if (signal_pending(current))
 268                         break;
 269                 timeout = MAX_SCHEDULE_TIMEOUT;
 270                 if (sk_wait_event(sk, &timeout, ctx->used, &wait)) {
 271                         err = 0;
 272                         break;
 273                 }
 274         }
 275         remove_wait_queue(sk_sleep(sk), &wait);
 276
 277         sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
 278
 279         return err;
 280 }
 281
 282 static void skcipher_data_wakeup(struct sock *sk)
 283 {
 284         struct alg_sock *ask = alg_sk(sk);
 285         struct skcipher_ctx *ctx = ask->private;
 286         struct socket_wq *wq;
 287
 288         if (!ctx->used)
 289                 return;
 290
 291         rcu_read_lock();
 292         wq = rcu_dereference(sk->sk_wq);
 293         if (skwq_has_sleeper(wq))
 294                 wake_up_interruptible_sync_poll(&wq->wait, POLLOUT |
 295                                                            POLLRDNORM |
 296                                                            POLLRDBAND);
 297         sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT);
 298         rcu_read_unlock();
 299 }
 300
 301 static int skcipher_sendmsg(struct socket *sock, struct msghdr *msg,
 302                             size_t size)
 303 {
 304         struct sock *sk = sock->sk;
 305         struct alg_sock *ask = alg_sk(sk);
 306         struct sock *psk = ask->parent;
 307         struct alg_sock *pask = alg_sk(psk);
 308         struct skcipher_ctx *ctx = ask->private;
 309         struct skcipher_tfm *skc = pask->private;
 310         struct crypto_skcipher *tfm = skc->skcipher;
 311         unsigned ivsize = crypto_skcipher_ivsize(tfm);
 312         struct skcipher_sg_list *sgl;
 313         struct af_alg_control con = {};
 314         long copied = 0;
 315         bool enc = 0;
 316         bool init = 0;
 317         int err;
 318         int i;
 319
 320         if (msg->msg_controllen) {
 321                 err = af_alg_cmsg_send(msg, &con);
 322                 if (err)
 323                         return err;
 324
 325                 init = 1;
 326                 switch (con.op) {
 327                 case ALG_OP_ENCRYPT:
 328                         enc = 1;
 329                         break;
 330                 case ALG_OP_DECRYPT:
 331                         enc = 0;
 332                         break;
 333                 default:
 334                         return -EINVAL;
 335                 }
 336
 337                 if (con.iv && con.iv->ivlen != ivsize)
 338                         return -EINVAL;
 339         }
 340
 341         err = -EINVAL;
 342
 343         lock_sock(sk);
 344         if (!ctx->more && ctx->used)
 345                 goto unlock;
 346
 347         if (init) {
 348                 ctx->enc = enc;
 349                 if (con.iv)
 350                         memcpy(ctx->iv, con.iv->iv, ivsize);
 351         }
 352
 353         while (size) {
 354                 struct scatterlist *sg;
 355                 unsigned long len = size;
 356                 size_t plen;
 357
 358                 if (ctx->merge) {
 359                         sgl = list_entry(ctx->tsgl.prev,
 360                                          struct skcipher_sg_list, list);
 361                         sg = sgl->sg + sgl->cur - 1;
 362                         len = min_t(unsigned long, len,
 363                                     PAGE_SIZE - sg->offset - sg->length);
 364
 365                         err = memcpy_from_msg(page_address(sg_page(sg)) +
 366                                               sg->offset + sg->length,
 367                                               msg, len);
 368                         if (err)
 369                                 goto unlock;
 370
 371                         sg->length += len;
 372                         ctx->merge = (sg->offset + sg->length) &
 373                                      (PAGE_SIZE - 1);
 374
 375                         ctx->used += len;
 376                         copied += len;
 377                         size -= len;
 378                         continue;
 379                 }
 380
 381                 if (!skcipher_writable(sk)) {
 382                         err = skcipher_wait_for_wmem(sk, msg->msg_flags);
 383                         if (err)
 384                                 goto unlock;
 385                 }
 386
 387                 len = min_t(unsigned long, len, skcipher_sndbuf(sk));
 388
 389                 err = skcipher_alloc_sgl(sk);
 390                 if (err)
 391                         goto unlock;
 392
 393                 sgl = list_entry(ctx->tsgl.prev, struct skcipher_sg_list, list);
 394                 sg = sgl->sg;
 395                 if (sgl->cur)
 396                         sg_unmark_end(sg + sgl->cur - 1);
 397                 do {
 398                         i = sgl->cur;
 399                         plen = min_t(size_t, len, PAGE_SIZE);
 400
 401                         sg_assign_page(sg + i, alloc_page(GFP_KERNEL));
 402                         err = -ENOMEM;
 403                         if (!sg_page(sg + i))
 404                                 goto unlock;
 405
 406                         err = memcpy_from_msg(page_address(sg_page(sg + i)),
 407                                               msg, plen);
 408                         if (err) {
 409                                 __free_page(sg_page(sg + i));
 410                                 sg_assign_page(sg + i, NULL);
 411                                 goto unlock;
 412                         }
 413
 414                         sg[i].length = plen;
 415                         len -= plen;
 416                         ctx->used += plen;
 417                         copied += plen;
 418                         size -= plen;
 419                         sgl->cur++;
 420                 } while (len && sgl->cur < MAX_SGL_ENTS);
 421
 422                 if (!size)
 423                         sg_mark_end(sg + sgl->cur - 1);
 424
 425                 ctx->merge = plen & (PAGE_SIZE - 1);
 426         }
 427
 428         err = 0;
 429
 430         ctx->more = msg->msg_flags & MSG_MORE;
 431
 432 unlock:
 433         skcipher_data_wakeup(sk);
 434         release_sock(sk);
 435
 436         return copied ?: err;
 437 }
 438
 439 static ssize_t skcipher_sendpage(struct socket *sock, struct page *page,
 440                                  int offset, size_t size, int flags)
 441 {
 442         struct sock *sk = sock->sk;
 443         struct alg_sock *ask = alg_sk(sk);
 444         struct skcipher_ctx *ctx = ask->private;
 445         struct skcipher_sg_list *sgl;
 446         int err = -EINVAL;
 447
 448         if (flags & MSG_SENDPAGE_NOTLAST)
 449                 flags |= MSG_MORE;
 450
 451         lock_sock(sk);
 452         if (!ctx->more && ctx->used)
 453                 goto unlock;
 454
 455         if (!size)
 456                 goto done;
 457
 458         if (!skcipher_writable(sk)) {
 459                 err = skcipher_wait_for_wmem(sk, flags);
 460                 if (err)
 461                         goto unlock;
 462         }
 463
 464         err = skcipher_alloc_sgl(sk);
 465         if (err)
 466                 goto unlock;
 467
 468         ctx->merge = 0;
 469         sgl = list_entry(ctx->tsgl.prev, struct skcipher_sg_list, list);
 470
 471         if (sgl->cur)
 472                 sg_unmark_end(sgl->sg + sgl->cur - 1);
 473
 474         sg_mark_end(sgl->sg + sgl->cur);
 475         get_page(page);
 476         sg_set_page(sgl->sg + sgl->cur, page, size, offset);
 477         sgl->cur++;
 478         ctx->used += size;
 479
 480 done:
 481         ctx->more = flags & MSG_MORE;
 482
 483 unlock:
 484         skcipher_data_wakeup(sk);
 485         release_sock(sk);
 486
 487         return err ?: size;
 488 }
 489
 490 static int skcipher_all_sg_nents(struct skcipher_ctx *ctx)
 491 {
 492         struct skcipher_sg_list *sgl;
 493         struct scatterlist *sg;
 494         int nents = 0;
 495
 496         list_for_each_entry(sgl, &ctx->tsgl, list) {
 497                 sg = sgl->sg;
 498
 499                 while (!sg->length)
 500                         sg++;
 501
 502                 nents += sg_nents(sg);
 503         }
 504         return nents;
 505 }
 506
 507 static int skcipher_recvmsg_async(struct socket *sock, struct msghdr *msg,
 508                                   int flags)
 509 {
 510         struct sock *sk = sock->sk;
 511         struct alg_sock *ask = alg_sk(sk);
 512         struct sock *psk = ask->parent;
 513         struct alg_sock *pask = alg_sk(psk);
 514         struct skcipher_ctx *ctx = ask->private;
 515         struct skcipher_tfm *skc = pask->private;
 516         struct crypto_skcipher *tfm = skc->skcipher;
 517         struct skcipher_sg_list *sgl;
 518         struct scatterlist *sg;
 519         struct skcipher_async_req *sreq;
 520         struct skcipher_request *req;
 521         struct skcipher_async_rsgl *last_rsgl = NULL;
 522         unsigned int txbufs = 0, len = 0, tx_nents;
 523         unsigned int reqsize = crypto_skcipher_reqsize(tfm);
 524         unsigned int ivsize = crypto_skcipher_ivsize(tfm);
 525         int err = -ENOMEM;
 526         bool mark = false;
 527         char *iv;
 528
 529         sreq = kzalloc(sizeof(*sreq) + reqsize + ivsize, GFP_KERNEL);
 530         if (unlikely(!sreq))
 531                 goto out;
 532
 533         req = &sreq->req;
 534         iv = (char *)(req + 1) + reqsize;
 535         sreq->iocb = msg->msg_iocb;
 536         INIT_LIST_HEAD(&sreq->list);
 537         sreq->inflight = &ctx->inflight;
 538
 539         lock_sock(sk);
 540         tx_nents = skcipher_all_sg_nents(ctx);
 541         sreq->tsg = kcalloc(tx_nents, sizeof(*sg), GFP_KERNEL);
 542         if (unlikely(!sreq->tsg))
 543                 goto unlock;
 544         sg_init_table(sreq->tsg, tx_nents);
 545         memcpy(iv, ctx->iv, ivsize);
 546         skcipher_request_set_tfm(req, tfm);
 547         skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP,
 548                                       skcipher_async_cb, sreq);
 549
 550         while (iov_iter_count(&msg->msg_iter)) {
 551                 struct skcipher_async_rsgl *rsgl;
 552                 int used;
 553
 554                 if (!ctx->used) {
 555                         err = skcipher_wait_for_data(sk, flags);
 556                         if (err)
 557                                 goto free;
 558                 }
 559                 sgl = list_first_entry(&ctx->tsgl,
 560                                        struct skcipher_sg_list, list);
 561                 sg = sgl->sg;
 562
 563                 while (!sg->length)
 564                         sg++;
 565
 566                 used = min_t(unsigned long, ctx->used,
 567                              iov_iter_count(&msg->msg_iter));
 568                 used = min_t(unsigned long, used, sg->length);
 569
 570                 if (txbufs == tx_nents) {
 571                         struct scatterlist *tmp;
 572                         int x;
 573                         /* Ran out of tx slots in async request
 574                          * need to expand */
 575                         tmp = kcalloc(tx_nents * 2, sizeof(*tmp),
 576                                       GFP_KERNEL);
 577                         if (!tmp) {
 578                                 err = -ENOMEM;
 579                                 goto free;
 580                         }
 581
 582                         sg_init_table(tmp, tx_nents * 2);
 583                         for (x = 0; x < tx_nents; x++)
 584                                 sg_set_page(&tmp[x], sg_page(&sreq->tsg[x]),
 585                                             sreq->tsg[x].length,
 586                                             sreq->tsg[x].offset);
 587                         kfree(sreq->tsg);
 588                         sreq->tsg = tmp;
 589                         tx_nents *= 2;
 590                         mark = true;
 591                 }
 592                 /* Need to take over the tx sgl from ctx
 593                  * to the asynch req - these sgls will be freed later */
 594                 sg_set_page(sreq->tsg + txbufs++, sg_page(sg), sg->length,
 595                             sg->offset);
 596
 597                 if (list_empty(&sreq->list)) {
 598                         rsgl = &sreq->first_sgl;
 599                         list_add_tail(&rsgl->list, &sreq->list);
 600                 } else {
 601                         rsgl = kmalloc(sizeof(*rsgl), GFP_KERNEL);
 602                         if (!rsgl) {
 603                                 err = -ENOMEM;
 604                                 goto free;
 605                         }
 606                         list_add_tail(&rsgl->list, &sreq->list);
 607                 }
 608
 609                 used = af_alg_make_sg(&rsgl->sgl, &msg->msg_iter, used);
 610                 err = used;
 611                 if (used < 0)
 612                         goto free;
 613                 if (last_rsgl)
 614                         af_alg_link_sg(&last_rsgl->sgl, &rsgl->sgl);
 615
 616                 last_rsgl = rsgl;
 617                 len += used;
 618                 skcipher_pull_sgl(sk, used, 0);
 619                 iov_iter_advance(&msg->msg_iter, used);
 620         }
 621
 622         if (mark)
 623                 sg_mark_end(sreq->tsg + txbufs - 1);
 624
 625         skcipher_request_set_crypt(req, sreq->tsg, sreq->first_sgl.sgl.sg,
 626                                    len, iv);
 627         err = ctx->enc ? crypto_skcipher_encrypt(req) :
 628                          crypto_skcipher_decrypt(req);
 629         if (err == -EINPROGRESS) {
 630                 atomic_inc(&ctx->inflight);
 631                 err = -EIOCBQUEUED;
 632                 sreq = NULL;
 633                 goto unlock;
 634         }
 635 free:
 636         skcipher_free_async_sgls(sreq);
 637 unlock:
 638         skcipher_wmem_wakeup(sk);
 639         release_sock(sk);
 640         kzfree(sreq);
 641 out:
 642         return err;
 643 }
 644
 645 static int skcipher_recvmsg_sync(struct socket *sock, struct msghdr *msg,
 646                                  int flags)
 647 {
 648         struct sock *sk = sock->sk;
 649         struct alg_sock *ask = alg_sk(sk);
 650         struct sock *psk = ask->parent;
 651         struct alg_sock *pask = alg_sk(psk);
 652         struct skcipher_ctx *ctx = ask->private;
 653         struct skcipher_tfm *skc = pask->private;
 654         struct crypto_skcipher *tfm = skc->skcipher;
 655         unsigned bs = crypto_skcipher_blocksize(tfm);
 656         struct skcipher_sg_list *sgl;
 657         struct scatterlist *sg;
 658         int err = -EAGAIN;
 659         int used;
 660         long copied = 0;
 661
 662         lock_sock(sk);
 663         while (msg_data_left(msg)) {
 664                 if (!ctx->used) {
 665                         err = skcipher_wait_for_data(sk, flags);
 666                         if (err)
 667                                 goto unlock;
 668                 }
 669
 670                 used = min_t(unsigned long, ctx->used, msg_data_left(msg));
 671
 672                 used = af_alg_make_sg(&ctx->rsgl, &msg->msg_iter, used);
 673                 err = used;
 674                 if (err < 0)
 675                         goto unlock;
 676
 677                 if (ctx->more || used < ctx->used)
 678                         used -= used % bs;
 679
 680                 err = -EINVAL;
 681                 if (!used)
 682                         goto free;
 683
 684                 sgl = list_first_entry(&ctx->tsgl,
 685                                        struct skcipher_sg_list, list);
 686                 sg = sgl->sg;
 687
 688                 while (!sg->length)
 689                         sg++;
 690
 691                 skcipher_request_set_crypt(&ctx->req, sg, ctx->rsgl.sg, used,
 692                                            ctx->iv);
 693
 694                 err = af_alg_wait_for_completion(
 695                                 ctx->enc ?
 696                                         crypto_skcipher_encrypt(&ctx->req) :
 697                                         crypto_skcipher_decrypt(&ctx->req),
 698                                 &ctx->completion);
 699
 700 free:
 701                 af_alg_free_sg(&ctx->rsgl);
 702
 703                 if (err)
 704                         goto unlock;
 705
 706                 copied += used;
 707                 skcipher_pull_sgl(sk, used, 1);
 708                 iov_iter_advance(&msg->msg_iter, used);
 709         }
 710
 711         err = 0;
 712
 713 unlock:
 714         skcipher_wmem_wakeup(sk);
 715         release_sock(sk);
 716
 717         return copied ?: err;
 718 }
 719
 720 static int skcipher_recvmsg(struct socket *sock, struct msghdr *msg,
 721                             size_t ignored, int flags)
 722 {
 723         return (msg->msg_iocb && !is_sync_kiocb(msg->msg_iocb)) ?
 724                 skcipher_recvmsg_async(sock, msg, flags) :
 725                 skcipher_recvmsg_sync(sock, msg, flags);
 726 }
 727
 728 static unsigned int skcipher_poll(struct file *file, struct socket *sock,
 729                                   poll_table *wait)
 730 {
 731         struct sock *sk = sock->sk;
 732         struct alg_sock *ask = alg_sk(sk);
 733         struct skcipher_ctx *ctx = ask->private;
 734         unsigned int mask;
 735
 736         sock_poll_wait(file, sk_sleep(sk), wait);
 737         mask = 0;
 738
 739         if (ctx->used)
 740                 mask |= POLLIN | POLLRDNORM;
 741
 742         if (skcipher_writable(sk))
 743                 mask |= POLLOUT | POLLWRNORM | POLLWRBAND;
 744
 745         return mask;
 746 }
 747
 748 static struct proto_ops algif_skcipher_ops = {
 749         .family         =       PF_ALG,
 750
 751         .connect        =       sock_no_connect,
 752         .socketpair     =       sock_no_socketpair,
 753         .getname        =       sock_no_getname,
 754         .ioctl          =       sock_no_ioctl,
 755         .listen         =       sock_no_listen,
 756         .shutdown       =       sock_no_shutdown,
 757         .getsockopt     =       sock_no_getsockopt,
 758         .mmap           =       sock_no_mmap,
 759         .bind           =       sock_no_bind,
 760         .accept         =       sock_no_accept,
 761         .setsockopt     =       sock_no_setsockopt,
 762
 763         .release        =       af_alg_release,
 764         .sendmsg        =       skcipher_sendmsg,
 765         .sendpage       =       skcipher_sendpage,
 766         .recvmsg        =       skcipher_recvmsg,
 767         .poll           =       skcipher_poll,
 768 };
 769
 770 static int skcipher_check_key(struct socket *sock)
 771 {
 772         int err = 0;
 773         struct sock *psk;
 774         struct alg_sock *pask;
 775         struct skcipher_tfm *tfm;
 776         struct sock *sk = sock->sk;
 777         struct alg_sock *ask = alg_sk(sk);
 778
 779         lock_sock(sk);
 780         if (ask->refcnt)
 781                 goto unlock_child;
 782
 783         psk = ask->parent;
 784         pask = alg_sk(ask->parent);
 785         tfm = pask->private;
 786
 787         err = -ENOKEY;
 788         lock_sock_nested(psk, SINGLE_DEPTH_NESTING);
 789         if (!tfm->has_key)
 790                 goto unlock;
 791
 792         if (!pask->refcnt++)
 793                 sock_hold(psk);
 794
 795         ask->refcnt = 1;
 796         sock_put(psk);
 797
 798         err = 0;
 799
 800 unlock:
 801         release_sock(psk);
 802 unlock_child:
 803         release_sock(sk);
 804
 805         return err;
 806 }
 807
 808 static int skcipher_sendmsg_nokey(struct socket *sock, struct msghdr *msg,
 809                                   size_t size)
 810 {
 811         int err;
 812
 813         err = skcipher_check_key(sock);
 814         if (err)
 815                 return err;
 816
 817         return skcipher_sendmsg(sock, msg, size);
 818 }
 819
 820 static ssize_t skcipher_sendpage_nokey(struct socket *sock, struct page *page,
 821                                        int offset, size_t size, int flags)
 822 {
 823         int err;
 824
 825         err = skcipher_check_key(sock);
 826         if (err)
 827                 return err;
 828
 829         return skcipher_sendpage(sock, page, offset, size, flags);
 830 }
 831
 832 static int skcipher_recvmsg_nokey(struct socket *sock, struct msghdr *msg,
 833                                   size_t ignored, int flags)
 834 {
 835         int err;
 836
 837         err = skcipher_check_key(sock);
 838         if (err)
 839                 return err;
 840
 841         return skcipher_recvmsg(sock, msg, ignored, flags);
 842 }
 843
 844 static struct proto_ops algif_skcipher_ops_nokey = {
 845         .family         =       PF_ALG,
 846
 847         .connect        =       sock_no_connect,
 848         .socketpair     =       sock_no_socketpair,
 849         .getname        =       sock_no_getname,
 850         .ioctl          =       sock_no_ioctl,
 851         .listen         =       sock_no_listen,
 852         .shutdown       =       sock_no_shutdown,
 853         .getsockopt     =       sock_no_getsockopt,
 854         .mmap           =       sock_no_mmap,
 855         .bind           =       sock_no_bind,
 856         .accept         =       sock_no_accept,
 857         .setsockopt     =       sock_no_setsockopt,
 858
 859         .release        =       af_alg_release,
 860         .sendmsg        =       skcipher_sendmsg_nokey,
 861         .sendpage       =       skcipher_sendpage_nokey,
 862         .recvmsg        =       skcipher_recvmsg_nokey,
 863         .poll           =       skcipher_poll,
 864 };
 865
 866 static void *skcipher_bind(const char *name, u32 type, u32 mask)
 867 {
 868         struct skcipher_tfm *tfm;
 869         struct crypto_skcipher *skcipher;
 870
 871         tfm = kzalloc(sizeof(*tfm), GFP_KERNEL);
 872         if (!tfm)
 873                 return ERR_PTR(-ENOMEM);
 874
 875         skcipher = crypto_alloc_skcipher(name, type, mask);
 876         if (IS_ERR(skcipher)) {
 877                 kfree(tfm);
 878                 return ERR_CAST(skcipher);
 879         }
 880
 881         tfm->skcipher = skcipher;
 882
 883         return tfm;
 884 }
 885
 886 static void skcipher_release(void *private)
 887 {
 888         struct skcipher_tfm *tfm = private;
 889
 890         crypto_free_skcipher(tfm->skcipher);
 891         kfree(tfm);
 892 }
 893
 894 static int skcipher_setkey(void *private, const u8 *key, unsigned int keylen)
 895 {
 896         struct skcipher_tfm *tfm = private;
 897         int err;
 898
 899         err = crypto_skcipher_setkey(tfm->skcipher, key, keylen);
 900         tfm->has_key = !err;
 901
 902         return err;
 903 }
 904
 905 static void skcipher_wait(struct sock *sk)
 906 {
 907         struct alg_sock *ask = alg_sk(sk);
 908         struct skcipher_ctx *ctx = ask->private;
 909         int ctr = 0;
 910
 911         while (atomic_read(&ctx->inflight) && ctr++ < 100)
 912                 msleep(100);
 913 }
 914
 915 static void skcipher_sock_destruct(struct sock *sk)
 916 {
 917         struct alg_sock *ask = alg_sk(sk);
 918         struct skcipher_ctx *ctx = ask->private;
 919         struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(&ctx->req);
 920
 921         if (atomic_read(&ctx->inflight))
 922                 skcipher_wait(sk);
 923
 924         skcipher_free_sgl(sk);
 925         sock_kzfree_s(sk, ctx->iv, crypto_skcipher_ivsize(tfm));
 926         sock_kfree_s(sk, ctx, ctx->len);
 927         af_alg_release_parent(sk);
 928 }
 929
 930 static int skcipher_accept_parent_nokey(void *private, struct sock *sk)
 931 {
 932         struct skcipher_ctx *ctx;
 933         struct alg_sock *ask = alg_sk(sk);
 934         struct skcipher_tfm *tfm = private;
 935         struct crypto_skcipher *skcipher = tfm->skcipher;
 936         unsigned int len = sizeof(*ctx) + crypto_skcipher_reqsize(skcipher);
 937
 938         ctx = sock_kmalloc(sk, len, GFP_KERNEL);
 939         if (!ctx)
 940                 return -ENOMEM;
 941
 942         ctx->iv = sock_kmalloc(sk, crypto_skcipher_ivsize(skcipher),
 943                                GFP_KERNEL);
 944         if (!ctx->iv) {
 945                 sock_kfree_s(sk, ctx, len);
 946                 return -ENOMEM;
 947         }
 948
 949         memset(ctx->iv, 0, crypto_skcipher_ivsize(skcipher));
 950
 951         INIT_LIST_HEAD(&ctx->tsgl);
 952         ctx->len = len;
 953         ctx->used = 0;
 954         ctx->more = 0;
 955         ctx->merge = 0;
 956         ctx->enc = 0;
 957         atomic_set(&ctx->inflight, 0);
 958         af_alg_init_completion(&ctx->completion);
 959
 960         ask->private = ctx;
 961
 962         skcipher_request_set_tfm(&ctx->req, skcipher);
 963         skcipher_request_set_callback(&ctx->req, CRYPTO_TFM_REQ_MAY_SLEEP |
 964                                                  CRYPTO_TFM_REQ_MAY_BACKLOG,
 965                                       af_alg_complete, &ctx->completion);
 966
 967         sk->sk_destruct = skcipher_sock_destruct;
 968
 969         return 0;
 970 }
 971
 972 static int skcipher_accept_parent(void *private, struct sock *sk)
 973 {
 974         struct skcipher_tfm *tfm = private;
 975
 976         if (!tfm->has_key && crypto_skcipher_has_setkey(tfm->skcipher))
 977                 return -ENOKEY;
 978
 979         return skcipher_accept_parent_nokey(private, sk);
 980 }
 981
 982 static const struct af_alg_type algif_type_skcipher = {
 983         .bind           =       skcipher_bind,
 984         .release        =       skcipher_release,
 985         .setkey         =       skcipher_setkey,
 986         .accept         =       skcipher_accept_parent,
 987         .accept_nokey   =       skcipher_accept_parent_nokey,
 988         .ops            =       &algif_skcipher_ops,
 989         .ops_nokey      =       &algif_skcipher_ops_nokey,
 990         .name           =       "skcipher",
 991         .owner          =       THIS_MODULE
 992 };
 993
 994 static int __init algif_skcipher_init(void)
 995 {
 996         return af_alg_register_type(&algif_type_skcipher);
 997 }
 998
 999 static void __exit algif_skcipher_exit(void)
1000 {
1001         int err = af_alg_unregister_type(&algif_type_skcipher);
1002         BUG_ON(err);
1003 }
1004
1005 module_init(algif_skcipher_init);
1006 module_exit(algif_skcipher_exit);
1007 MODULE_LICENSE("GPL");