crypto/pcrypt.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * pcrypt - Parallel crypto wrapper.
   4  *
   5  * Copyright (C) 2009 secunet Security Networks AG
   6  * Copyright (C) 2009 Steffen Klassert <steffen.klassert@secunet.com>
   7  */
   8
   9 #include <crypto/algapi.h>
  10 #include <crypto/internal/aead.h>
  11 #include <linux/atomic.h>
  12 #include <linux/err.h>
  13 #include <linux/init.h>
  14 #include <linux/module.h>
  15 #include <linux/slab.h>
  16 #include <linux/notifier.h>
  17 #include <linux/kobject.h>
  18 #include <linux/cpu.h>
  19 #include <crypto/pcrypt.h>
  20
  21 static struct padata_instance *pencrypt;
  22 static struct padata_instance *pdecrypt;
  23 static struct kset           *pcrypt_kset;
  24
  25 struct pcrypt_instance_ctx {
  26         struct crypto_aead_spawn spawn;
  27         struct padata_shell *psenc;
  28         struct padata_shell *psdec;
  29         atomic_t tfm_count;
  30 };
  31
  32 struct pcrypt_aead_ctx {
  33         struct crypto_aead *child;
  34         unsigned int cb_cpu;
  35 };
  36
  37 static inline struct pcrypt_instance_ctx *pcrypt_tfm_ictx(
  38         struct crypto_aead *tfm)
  39 {
  40         return aead_instance_ctx(aead_alg_instance(tfm));
  41 }
  42
  43 static int pcrypt_aead_setkey(struct crypto_aead *parent,
  44                               const u8 *key, unsigned int keylen)
  45 {
  46         struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(parent);
  47
  48         return crypto_aead_setkey(ctx->child, key, keylen);
  49 }
  50
  51 static int pcrypt_aead_setauthsize(struct crypto_aead *parent,
  52                                    unsigned int authsize)
  53 {
  54         struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(parent);
  55
  56         return crypto_aead_setauthsize(ctx->child, authsize);
  57 }
  58
  59 static void pcrypt_aead_serial(struct padata_priv *padata)
  60 {
  61         struct pcrypt_request *preq = pcrypt_padata_request(padata);
  62         struct aead_request *req = pcrypt_request_ctx(preq);
  63
  64         aead_request_complete(req->base.data, padata->info);
  65 }
  66
  67 static void pcrypt_aead_done(struct crypto_async_request *areq, int err)
  68 {
  69         struct aead_request *req = areq->data;
  70         struct pcrypt_request *preq = aead_request_ctx(req);
  71         struct padata_priv *padata = pcrypt_request_padata(preq);
  72
  73         padata->info = err;
  74         req->base.flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
  75
  76         padata_do_serial(padata);
  77 }
  78
  79 static void pcrypt_aead_enc(struct padata_priv *padata)
  80 {
  81         struct pcrypt_request *preq = pcrypt_padata_request(padata);
  82         struct aead_request *req = pcrypt_request_ctx(preq);
  83
  84         padata->info = crypto_aead_encrypt(req);
  85
  86         if (padata->info == -EINPROGRESS)
  87                 return;
  88
  89         padata_do_serial(padata);
  90 }
  91
  92 static int pcrypt_aead_encrypt(struct aead_request *req)
  93 {
  94         int err;
  95         struct pcrypt_request *preq = aead_request_ctx(req);
  96         struct aead_request *creq = pcrypt_request_ctx(preq);
  97         struct padata_priv *padata = pcrypt_request_padata(preq);
  98         struct crypto_aead *aead = crypto_aead_reqtfm(req);
  99         struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(aead);
 100         u32 flags = aead_request_flags(req);
 101         struct pcrypt_instance_ctx *ictx;
 102
 103         ictx = pcrypt_tfm_ictx(aead);
 104
 105         memset(padata, 0, sizeof(struct padata_priv));
 106
 107         padata->parallel = pcrypt_aead_enc;
 108         padata->serial = pcrypt_aead_serial;
 109
 110         aead_request_set_tfm(creq, ctx->child);
 111         aead_request_set_callback(creq, flags & ~CRYPTO_TFM_REQ_MAY_SLEEP,
 112                                   pcrypt_aead_done, req);
 113         aead_request_set_crypt(creq, req->src, req->dst,
 114                                req->cryptlen, req->iv);
 115         aead_request_set_ad(creq, req->assoclen);
 116
 117         err = padata_do_parallel(ictx->psenc, padata, &ctx->cb_cpu);
 118         if (!err)
 119                 return -EINPROGRESS;
 120
 121         return err;
 122 }
 123
 124 static void pcrypt_aead_dec(struct padata_priv *padata)
 125 {
 126         struct pcrypt_request *preq = pcrypt_padata_request(padata);
 127         struct aead_request *req = pcrypt_request_ctx(preq);
 128
 129         padata->info = crypto_aead_decrypt(req);
 130
 131         if (padata->info == -EINPROGRESS)
 132                 return;
 133
 134         padata_do_serial(padata);
 135 }
 136
 137 static int pcrypt_aead_decrypt(struct aead_request *req)
 138 {
 139         int err;
 140         struct pcrypt_request *preq = aead_request_ctx(req);
 141         struct aead_request *creq = pcrypt_request_ctx(preq);
 142         struct padata_priv *padata = pcrypt_request_padata(preq);
 143         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 144         struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(aead);
 145         u32 flags = aead_request_flags(req);
 146         struct pcrypt_instance_ctx *ictx;
 147
 148         ictx = pcrypt_tfm_ictx(aead);
 149
 150         memset(padata, 0, sizeof(struct padata_priv));
 151
 152         padata->parallel = pcrypt_aead_dec;
 153         padata->serial = pcrypt_aead_serial;
 154
 155         aead_request_set_tfm(creq, ctx->child);
 156         aead_request_set_callback(creq, flags & ~CRYPTO_TFM_REQ_MAY_SLEEP,
 157                                   pcrypt_aead_done, req);
 158         aead_request_set_crypt(creq, req->src, req->dst,
 159                                req->cryptlen, req->iv);
 160         aead_request_set_ad(creq, req->assoclen);
 161
 162         err = padata_do_parallel(ictx->psdec, padata, &ctx->cb_cpu);
 163         if (!err)
 164                 return -EINPROGRESS;
 165
 166         return err;
 167 }
 168
 169 static int pcrypt_aead_init_tfm(struct crypto_aead *tfm)
 170 {
 171         int cpu, cpu_index;
 172         struct aead_instance *inst = aead_alg_instance(tfm);
 173         struct pcrypt_instance_ctx *ictx = aead_instance_ctx(inst);
 174         struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(tfm);
 175         struct crypto_aead *cipher;
 176
 177         cpu_index = (unsigned int)atomic_inc_return(&ictx->tfm_count) %
 178                     cpumask_weight(cpu_online_mask);
 179
 180         ctx->cb_cpu = cpumask_first(cpu_online_mask);
 181         for (cpu = 0; cpu < cpu_index; cpu++)
 182                 ctx->cb_cpu = cpumask_next(ctx->cb_cpu, cpu_online_mask);
 183
 184         cipher = crypto_spawn_aead(&ictx->spawn);
 185
 186         if (IS_ERR(cipher))
 187                 return PTR_ERR(cipher);
 188
 189         ctx->child = cipher;
 190         crypto_aead_set_reqsize(tfm, sizeof(struct pcrypt_request) +
 191                                      sizeof(struct aead_request) +
 192                                      crypto_aead_reqsize(cipher));
 193
 194         return 0;
 195 }
 196
 197 static void pcrypt_aead_exit_tfm(struct crypto_aead *tfm)
 198 {
 199         struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(tfm);
 200
 201         crypto_free_aead(ctx->child);
 202 }
 203
 204 static void pcrypt_free(struct aead_instance *inst)
 205 {
 206         struct pcrypt_instance_ctx *ctx = aead_instance_ctx(inst);
 207
 208         crypto_drop_aead(&ctx->spawn);
 209         padata_free_shell(ctx->psdec);
 210         padata_free_shell(ctx->psenc);
 211         kfree(inst);
 212 }
 213
 214 static int pcrypt_init_instance(struct crypto_instance *inst,
 215                                 struct crypto_alg *alg)
 216 {
 217         if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
 218                      "pcrypt(%s)", alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME)
 219                 return -ENAMETOOLONG;
 220
 221         memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME);
 222
 223         inst->alg.cra_priority = alg->cra_priority + 100;
 224         inst->alg.cra_blocksize = alg->cra_blocksize;
 225         inst->alg.cra_alignmask = alg->cra_alignmask;
 226
 227         return 0;
 228 }
 229
 230 static int pcrypt_create_aead(struct crypto_template *tmpl, struct rtattr **tb,
 231                               u32 type, u32 mask)
 232 {
 233         struct pcrypt_instance_ctx *ctx;
 234         struct crypto_attr_type *algt;
 235         struct aead_instance *inst;
 236         struct aead_alg *alg;
 237         const char *name;
 238         int err;
 239
 240         algt = crypto_get_attr_type(tb);
 241         if (IS_ERR(algt))
 242                 return PTR_ERR(algt);
 243
 244         name = crypto_attr_alg_name(tb[1]);
 245         if (IS_ERR(name))
 246                 return PTR_ERR(name);
 247
 248         inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
 249         if (!inst)
 250                 return -ENOMEM;
 251
 252         err = -ENOMEM;
 253
 254         ctx = aead_instance_ctx(inst);
 255         ctx->psenc = padata_alloc_shell(pencrypt);
 256         if (!ctx->psenc)
 257                 goto out_free_inst;
 258
 259         ctx->psdec = padata_alloc_shell(pdecrypt);
 260         if (!ctx->psdec)
 261                 goto out_free_psenc;
 262
 263         crypto_set_aead_spawn(&ctx->spawn, aead_crypto_instance(inst));
 264
 265         err = crypto_grab_aead(&ctx->spawn, name, 0, 0);
 266         if (err)
 267                 goto out_free_psdec;
 268
 269         alg = crypto_spawn_aead_alg(&ctx->spawn);
 270         err = pcrypt_init_instance(aead_crypto_instance(inst), &alg->base);
 271         if (err)
 272                 goto out_drop_aead;
 273
 274         inst->alg.base.cra_flags = CRYPTO_ALG_ASYNC;
 275
 276         inst->alg.ivsize = crypto_aead_alg_ivsize(alg);
 277         inst->alg.maxauthsize = crypto_aead_alg_maxauthsize(alg);
 278
 279         inst->alg.base.cra_ctxsize = sizeof(struct pcrypt_aead_ctx);
 280
 281         inst->alg.init = pcrypt_aead_init_tfm;
 282         inst->alg.exit = pcrypt_aead_exit_tfm;
 283
 284         inst->alg.setkey = pcrypt_aead_setkey;
 285         inst->alg.setauthsize = pcrypt_aead_setauthsize;
 286         inst->alg.encrypt = pcrypt_aead_encrypt;
 287         inst->alg.decrypt = pcrypt_aead_decrypt;
 288
 289         inst->free = pcrypt_free;
 290
 291         err = aead_register_instance(tmpl, inst);
 292         if (err)
 293                 goto out_drop_aead;
 294
 295 out:
 296         return err;
 297
 298 out_drop_aead:
 299         crypto_drop_aead(&ctx->spawn);
 300 out_free_psdec:
 301         padata_free_shell(ctx->psdec);
 302 out_free_psenc:
 303         padata_free_shell(ctx->psenc);
 304 out_free_inst:
 305         kfree(inst);
 306         goto out;
 307 }
 308
 309 static int pcrypt_create(struct crypto_template *tmpl, struct rtattr **tb)
 310 {
 311         struct crypto_attr_type *algt;
 312
 313         algt = crypto_get_attr_type(tb);
 314         if (IS_ERR(algt))
 315                 return PTR_ERR(algt);
 316
 317         switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) {
 318         case CRYPTO_ALG_TYPE_AEAD:
 319                 return pcrypt_create_aead(tmpl, tb, algt->type, algt->mask);
 320         }
 321
 322         return -EINVAL;
 323 }
 324
 325 static int pcrypt_sysfs_add(struct padata_instance *pinst, const char *name)
 326 {
 327         int ret;
 328
 329         pinst->kobj.kset = pcrypt_kset;
 330         ret = kobject_add(&pinst->kobj, NULL, "%s", name);
 331         if (!ret)
 332                 kobject_uevent(&pinst->kobj, KOBJ_ADD);
 333
 334         return ret;
 335 }
 336
 337 static int pcrypt_init_padata(struct padata_instance **pinst, const char *name)
 338 {
 339         int ret = -ENOMEM;
 340
 341         *pinst = padata_alloc_possible(name);
 342         if (!*pinst)
 343                 return ret;
 344
 345         ret = pcrypt_sysfs_add(*pinst, name);
 346         if (ret)
 347                 padata_free(*pinst);
 348
 349         return ret;
 350 }
 351
 352 static void pcrypt_fini_padata(struct padata_instance *pinst)
 353 {
 354         padata_stop(pinst);
 355         padata_free(pinst);
 356 }
 357
 358 static struct crypto_template pcrypt_tmpl = {
 359         .name = "pcrypt",
 360         .create = pcrypt_create,
 361         .module = THIS_MODULE,
 362 };
 363
 364 static int __init pcrypt_init(void)
 365 {
 366         int err = -ENOMEM;
 367
 368         pcrypt_kset = kset_create_and_add("pcrypt", NULL, kernel_kobj);
 369         if (!pcrypt_kset)
 370                 goto err;
 371
 372         err = pcrypt_init_padata(&pencrypt, "pencrypt");
 373         if (err)
 374                 goto err_unreg_kset;
 375
 376         err = pcrypt_init_padata(&pdecrypt, "pdecrypt");
 377         if (err)
 378                 goto err_deinit_pencrypt;
 379
 380         padata_start(pencrypt);
 381         padata_start(pdecrypt);
 382
 383         return crypto_register_template(&pcrypt_tmpl);
 384
 385 err_deinit_pencrypt:
 386         pcrypt_fini_padata(pencrypt);
 387 err_unreg_kset:
 388         kset_unregister(pcrypt_kset);
 389 err:
 390         return err;
 391 }
 392
 393 static void __exit pcrypt_exit(void)
 394 {
 395         crypto_unregister_template(&pcrypt_tmpl);
 396
 397         pcrypt_fini_padata(pencrypt);
 398         pcrypt_fini_padata(pdecrypt);
 399
 400         kset_unregister(pcrypt_kset);
 401 }
 402
 403 subsys_initcall(pcrypt_init);
 404 module_exit(pcrypt_exit);
 405
 406 MODULE_LICENSE("GPL");
 407 MODULE_AUTHOR("Steffen Klassert <steffen.klassert@secunet.com>");
 408 MODULE_DESCRIPTION("Parallel crypto wrapper");
 409 MODULE_ALIAS_CRYPTO("pcrypt");