2 * Routines supporting the Power 7+ Nest Accelerators driver
4 * Copyright (C) 2011-2012 International Business Machines Inc.
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; version 2 only.
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
19 * Author: Kent Yoder <yoder1@us.ibm.com>
22 #include <crypto/internal/hash.h>
23 #include <crypto/hash.h>
24 #include <crypto/aes.h>
25 #include <crypto/sha.h>
26 #include <crypto/algapi.h>
27 #include <crypto/scatterwalk.h>
28 #include <linux/module.h>
29 #include <linux/moduleparam.h>
30 #include <linux/types.h>
32 #include <linux/crypto.h>
33 #include <linux/scatterlist.h>
34 #include <linux/device.h>
36 #include <asm/hvcall.h>
39 #include "nx_csbcpb.h"
44 * nx_hcall_sync - make an H_COP_OP hcall for the passed in op structure
46 * @nx_ctx: the crypto context handle
47 * @op: PFO operation struct to pass in
48 * @may_sleep: flag indicating the request can sleep
50 * Make the hcall, retrying while the hardware is busy. If we cannot yield
51 * the thread, limit the number of retries to 10 here.
53 int nx_hcall_sync(struct nx_crypto_ctx
*nx_ctx
,
54 struct vio_pfo_op
*op
,
58 struct vio_dev
*viodev
= nx_driver
.viodev
;
60 atomic_inc(&(nx_ctx
->stats
->sync_ops
));
63 rc
= vio_h_cop_sync(viodev
, op
);
64 } while ((rc
== -EBUSY
&& !may_sleep
&& retries
--) ||
65 (rc
== -EBUSY
&& may_sleep
&& cond_resched()));
68 dev_dbg(&viodev
->dev
, "vio_h_cop_sync failed: rc: %d "
69 "hcall rc: %ld\n", rc
, op
->hcall_err
);
70 atomic_inc(&(nx_ctx
->stats
->errors
));
71 atomic_set(&(nx_ctx
->stats
->last_error
), op
->hcall_err
);
72 atomic_set(&(nx_ctx
->stats
->last_error_pid
), current
->pid
);
79 * nx_build_sg_list - build an NX scatter list describing a single buffer
81 * @sg_head: pointer to the first scatter list element to build
82 * @start_addr: pointer to the linear buffer
83 * @len: length of the data at @start_addr
84 * @sgmax: the largest number of scatter list elements we're allowed to create
86 * This function will start writing nx_sg elements at @sg_head and keep
87 * writing them until all of the data from @start_addr is described or
88 * until sgmax elements have been written. Scatter list elements will be
89 * created such that none of the elements describes a buffer that crosses a 4K
92 struct nx_sg
*nx_build_sg_list(struct nx_sg
*sg_head
,
97 unsigned int sg_len
= 0;
99 u64 sg_addr
= (u64
)start_addr
;
102 /* determine the start and end for this address range - slightly
103 * different if this is in VMALLOC_REGION */
104 if (is_vmalloc_addr(start_addr
))
105 sg_addr
= page_to_phys(vmalloc_to_page(start_addr
))
106 + offset_in_page(sg_addr
);
108 sg_addr
= __pa(sg_addr
);
110 end_addr
= sg_addr
+ len
;
112 /* each iteration will write one struct nx_sg element and add the
113 * length of data described by that element to sg_len. Once @len bytes
114 * have been described (or @sgmax elements have been written), the
115 * loop ends. min_t is used to ensure @end_addr falls on the same page
116 * as sg_addr, if not, we need to create another nx_sg element for the
117 * data on the next page */
118 for (sg
= sg_head
; sg_len
< len
; sg
++) {
120 sg_addr
= min_t(u64
, NX_PAGE_NUM(sg_addr
+ NX_PAGE_SIZE
), end_addr
);
121 sg
->len
= sg_addr
- sg
->addr
;
124 if ((sg
- sg_head
) == sgmax
) {
125 pr_err("nx: scatter/gather list overflow, pid: %d\n",
131 /* return the moved sg_head pointer */
136 * nx_walk_and_build - walk a linux scatterlist and build an nx scatterlist
138 * @nx_dst: pointer to the first nx_sg element to write
139 * @sglen: max number of nx_sg entries we're allowed to write
140 * @sg_src: pointer to the source linux scatterlist to walk
141 * @start: number of bytes to fast-forward past at the beginning of @sg_src
142 * @src_len: number of bytes to walk in @sg_src
144 struct nx_sg
*nx_walk_and_build(struct nx_sg
*nx_dst
,
146 struct scatterlist
*sg_src
,
148 unsigned int src_len
)
150 struct scatter_walk walk
;
151 struct nx_sg
*nx_sg
= nx_dst
;
152 unsigned int n
, offset
= 0, len
= src_len
;
155 /* we need to fast forward through @start bytes first */
157 scatterwalk_start(&walk
, sg_src
);
159 if (start
< offset
+ sg_src
->length
)
162 offset
+= sg_src
->length
;
163 sg_src
= scatterwalk_sg_next(sg_src
);
166 /* start - offset is the number of bytes to advance in the scatterlist
167 * element we're currently looking at */
168 scatterwalk_advance(&walk
, start
- offset
);
170 while (len
&& nx_sg
) {
171 n
= scatterwalk_clamp(&walk
, len
);
173 scatterwalk_start(&walk
, sg_next(walk
.sg
));
174 n
= scatterwalk_clamp(&walk
, len
);
176 dst
= scatterwalk_map(&walk
);
178 nx_sg
= nx_build_sg_list(nx_sg
, dst
, n
, sglen
);
181 scatterwalk_unmap(dst
);
182 scatterwalk_advance(&walk
, n
);
183 scatterwalk_done(&walk
, SCATTERWALK_FROM_SG
, len
);
186 /* return the moved destination pointer */
191 * nx_build_sg_lists - walk the input scatterlists and build arrays of NX
192 * scatterlists based on them.
194 * @nx_ctx: NX crypto context for the lists we're building
195 * @desc: the block cipher descriptor for the operation
196 * @dst: destination scatterlist
197 * @src: source scatterlist
198 * @nbytes: length of data described in the scatterlists
199 * @iv: destination for the iv data, if the algorithm requires it
201 * This is common code shared by all the AES algorithms. It uses the block
202 * cipher walk routines to traverse input and output scatterlists, building
203 * corresponding NX scatterlists
205 int nx_build_sg_lists(struct nx_crypto_ctx
*nx_ctx
,
206 struct blkcipher_desc
*desc
,
207 struct scatterlist
*dst
,
208 struct scatterlist
*src
,
212 struct nx_sg
*nx_insg
= nx_ctx
->in_sg
;
213 struct nx_sg
*nx_outsg
= nx_ctx
->out_sg
;
214 struct blkcipher_walk walk
;
217 blkcipher_walk_init(&walk
, dst
, src
, nbytes
);
218 rc
= blkcipher_walk_virt_block(desc
, &walk
, AES_BLOCK_SIZE
);
223 memcpy(iv
, walk
.iv
, AES_BLOCK_SIZE
);
225 while (walk
.nbytes
) {
226 nx_insg
= nx_build_sg_list(nx_insg
, walk
.src
.virt
.addr
,
227 walk
.nbytes
, nx_ctx
->ap
->sglen
);
228 nx_outsg
= nx_build_sg_list(nx_outsg
, walk
.dst
.virt
.addr
,
229 walk
.nbytes
, nx_ctx
->ap
->sglen
);
231 rc
= blkcipher_walk_done(desc
, &walk
, 0);
237 nx_insg
= nx_build_sg_list(nx_insg
, walk
.src
.virt
.addr
,
238 walk
.nbytes
, nx_ctx
->ap
->sglen
);
239 nx_outsg
= nx_build_sg_list(nx_outsg
, walk
.dst
.virt
.addr
,
240 walk
.nbytes
, nx_ctx
->ap
->sglen
);
245 /* these lengths should be negative, which will indicate to phyp that
246 * the input and output parameters are scatterlists, not linear
248 nx_ctx
->op
.inlen
= (nx_ctx
->in_sg
- nx_insg
) * sizeof(struct nx_sg
);
249 nx_ctx
->op
.outlen
= (nx_ctx
->out_sg
- nx_outsg
) * sizeof(struct nx_sg
);
255 * nx_ctx_init - initialize an nx_ctx's vio_pfo_op struct
257 * @nx_ctx: the nx context to initialize
258 * @function: the function code for the op
260 void nx_ctx_init(struct nx_crypto_ctx
*nx_ctx
, unsigned int function
)
262 memset(nx_ctx
->kmem
, 0, nx_ctx
->kmem_len
);
263 nx_ctx
->csbcpb
->csb
.valid
|= NX_CSB_VALID_BIT
;
265 nx_ctx
->op
.flags
= function
;
266 nx_ctx
->op
.csbcpb
= __pa(nx_ctx
->csbcpb
);
267 nx_ctx
->op
.in
= __pa(nx_ctx
->in_sg
);
268 nx_ctx
->op
.out
= __pa(nx_ctx
->out_sg
);
270 if (nx_ctx
->csbcpb_aead
) {
271 nx_ctx
->csbcpb_aead
->csb
.valid
|= NX_CSB_VALID_BIT
;
273 nx_ctx
->op_aead
.flags
= function
;
274 nx_ctx
->op_aead
.csbcpb
= __pa(nx_ctx
->csbcpb_aead
);
275 nx_ctx
->op_aead
.in
= __pa(nx_ctx
->in_sg
);
276 nx_ctx
->op_aead
.out
= __pa(nx_ctx
->out_sg
);
280 static void nx_of_update_status(struct device
*dev
,
284 if (!strncmp(p
->value
, "okay", p
->length
)) {
285 props
->status
= NX_WAITING
;
286 props
->flags
|= NX_OF_FLAG_STATUS_SET
;
288 dev_info(dev
, "%s: status '%s' is not 'okay'\n", __func__
,
293 static void nx_of_update_sglen(struct device
*dev
,
297 if (p
->length
!= sizeof(props
->max_sg_len
)) {
298 dev_err(dev
, "%s: unexpected format for "
299 "ibm,max-sg-len property\n", __func__
);
300 dev_dbg(dev
, "%s: ibm,max-sg-len is %d bytes "
301 "long, expected %zd bytes\n", __func__
,
302 p
->length
, sizeof(props
->max_sg_len
));
306 props
->max_sg_len
= *(u32
*)p
->value
;
307 props
->flags
|= NX_OF_FLAG_MAXSGLEN_SET
;
310 static void nx_of_update_msc(struct device
*dev
,
314 struct msc_triplet
*trip
;
315 struct max_sync_cop
*msc
;
316 unsigned int bytes_so_far
, i
, lenp
;
318 msc
= (struct max_sync_cop
*)p
->value
;
321 /* You can't tell if the data read in for this property is sane by its
322 * size alone. This is because there are sizes embedded in the data
323 * structure. The best we can do is check lengths as we parse and bail
324 * as soon as a length error is detected. */
327 while ((bytes_so_far
+ sizeof(struct max_sync_cop
)) <= lenp
) {
328 bytes_so_far
+= sizeof(struct max_sync_cop
);
333 ((bytes_so_far
+ sizeof(struct msc_triplet
)) <= lenp
) &&
336 if (msc
->fc
> NX_MAX_FC
|| msc
->mode
> NX_MAX_MODE
) {
337 dev_err(dev
, "unknown function code/mode "
338 "combo: %d/%d (ignored)\n", msc
->fc
,
343 switch (trip
->keybitlen
) {
346 props
->ap
[msc
->fc
][msc
->mode
][0].databytelen
=
348 props
->ap
[msc
->fc
][msc
->mode
][0].sglen
=
352 props
->ap
[msc
->fc
][msc
->mode
][1].databytelen
=
354 props
->ap
[msc
->fc
][msc
->mode
][1].sglen
=
358 if (msc
->fc
== NX_FC_AES
) {
359 props
->ap
[msc
->fc
][msc
->mode
][2].
360 databytelen
= trip
->databytelen
;
361 props
->ap
[msc
->fc
][msc
->mode
][2].sglen
=
363 } else if (msc
->fc
== NX_FC_AES_HMAC
||
364 msc
->fc
== NX_FC_SHA
) {
365 props
->ap
[msc
->fc
][msc
->mode
][1].
366 databytelen
= trip
->databytelen
;
367 props
->ap
[msc
->fc
][msc
->mode
][1].sglen
=
370 dev_warn(dev
, "unknown function "
371 "code/key bit len combo"
372 ": (%u/256)\n", msc
->fc
);
376 props
->ap
[msc
->fc
][msc
->mode
][2].databytelen
=
378 props
->ap
[msc
->fc
][msc
->mode
][2].sglen
=
382 dev_warn(dev
, "unknown function code/key bit "
383 "len combo: (%u/%u)\n", msc
->fc
,
388 bytes_so_far
+= sizeof(struct msc_triplet
);
392 msc
= (struct max_sync_cop
*)trip
;
395 props
->flags
|= NX_OF_FLAG_MAXSYNCCOP_SET
;
399 * nx_of_init - read openFirmware values from the device tree
401 * @dev: device handle
402 * @props: pointer to struct to hold the properties values
404 * Called once at driver probe time, this function will read out the
405 * openFirmware properties we use at runtime. If all the OF properties are
406 * acceptable, when we exit this function props->flags will indicate that
407 * we're ready to register our crypto algorithms.
409 static void nx_of_init(struct device
*dev
, struct nx_of
*props
)
411 struct device_node
*base_node
= dev
->of_node
;
414 p
= of_find_property(base_node
, "status", NULL
);
416 dev_info(dev
, "%s: property 'status' not found\n", __func__
);
418 nx_of_update_status(dev
, p
, props
);
420 p
= of_find_property(base_node
, "ibm,max-sg-len", NULL
);
422 dev_info(dev
, "%s: property 'ibm,max-sg-len' not found\n",
425 nx_of_update_sglen(dev
, p
, props
);
427 p
= of_find_property(base_node
, "ibm,max-sync-cop", NULL
);
429 dev_info(dev
, "%s: property 'ibm,max-sync-cop' not found\n",
432 nx_of_update_msc(dev
, p
, props
);
436 * nx_register_algs - register algorithms with the crypto API
438 * Called from nx_probe()
440 * If all OF properties are in an acceptable state, the driver flags will
441 * indicate that we're ready and we'll create our debugfs files and register
442 * out crypto algorithms.
444 static int nx_register_algs(void)
448 if (nx_driver
.of
.flags
!= NX_OF_FLAG_MASK_READY
)
451 memset(&nx_driver
.stats
, 0, sizeof(struct nx_stats
));
453 rc
= NX_DEBUGFS_INIT(&nx_driver
);
457 rc
= crypto_register_alg(&nx_ecb_aes_alg
);
461 rc
= crypto_register_alg(&nx_cbc_aes_alg
);
465 rc
= crypto_register_alg(&nx_ctr_aes_alg
);
469 rc
= crypto_register_alg(&nx_ctr3686_aes_alg
);
473 rc
= crypto_register_alg(&nx_gcm_aes_alg
);
475 goto out_unreg_ctr3686
;
477 rc
= crypto_register_alg(&nx_gcm4106_aes_alg
);
481 rc
= crypto_register_alg(&nx_ccm_aes_alg
);
483 goto out_unreg_gcm4106
;
485 rc
= crypto_register_alg(&nx_ccm4309_aes_alg
);
489 rc
= crypto_register_shash(&nx_shash_sha256_alg
);
491 goto out_unreg_ccm4309
;
493 rc
= crypto_register_shash(&nx_shash_sha512_alg
);
497 rc
= crypto_register_shash(&nx_shash_aes_xcbc_alg
);
501 nx_driver
.of
.status
= NX_OKAY
;
506 crypto_unregister_shash(&nx_shash_sha512_alg
);
508 crypto_unregister_shash(&nx_shash_sha256_alg
);
510 crypto_unregister_alg(&nx_ccm4309_aes_alg
);
512 crypto_unregister_alg(&nx_ccm_aes_alg
);
514 crypto_unregister_alg(&nx_gcm4106_aes_alg
);
516 crypto_unregister_alg(&nx_gcm_aes_alg
);
518 crypto_unregister_alg(&nx_ctr3686_aes_alg
);
520 crypto_unregister_alg(&nx_ctr_aes_alg
);
522 crypto_unregister_alg(&nx_cbc_aes_alg
);
524 crypto_unregister_alg(&nx_ecb_aes_alg
);
530 * nx_crypto_ctx_init - create and initialize a crypto api context
532 * @nx_ctx: the crypto api context
533 * @fc: function code for the context
534 * @mode: the function code specific mode for this context
536 static int nx_crypto_ctx_init(struct nx_crypto_ctx
*nx_ctx
, u32 fc
, u32 mode
)
538 if (nx_driver
.of
.status
!= NX_OKAY
) {
539 pr_err("Attempt to initialize NX crypto context while device "
540 "is not available!\n");
544 /* we need an extra page for csbcpb_aead for these modes */
545 if (mode
== NX_MODE_AES_GCM
|| mode
== NX_MODE_AES_CCM
)
546 nx_ctx
->kmem_len
= (4 * NX_PAGE_SIZE
) +
547 sizeof(struct nx_csbcpb
);
549 nx_ctx
->kmem_len
= (3 * NX_PAGE_SIZE
) +
550 sizeof(struct nx_csbcpb
);
552 nx_ctx
->kmem
= kmalloc(nx_ctx
->kmem_len
, GFP_KERNEL
);
556 /* the csbcpb and scatterlists must be 4K aligned pages */
557 nx_ctx
->csbcpb
= (struct nx_csbcpb
*)(round_up((u64
)nx_ctx
->kmem
,
559 nx_ctx
->in_sg
= (struct nx_sg
*)((u8
*)nx_ctx
->csbcpb
+ NX_PAGE_SIZE
);
560 nx_ctx
->out_sg
= (struct nx_sg
*)((u8
*)nx_ctx
->in_sg
+ NX_PAGE_SIZE
);
562 if (mode
== NX_MODE_AES_GCM
|| mode
== NX_MODE_AES_CCM
)
563 nx_ctx
->csbcpb_aead
=
564 (struct nx_csbcpb
*)((u8
*)nx_ctx
->out_sg
+
567 /* give each context a pointer to global stats and their OF
569 nx_ctx
->stats
= &nx_driver
.stats
;
570 memcpy(nx_ctx
->props
, nx_driver
.of
.ap
[fc
][mode
],
571 sizeof(struct alg_props
) * 3);
576 /* entry points from the crypto tfm initializers */
577 int nx_crypto_ctx_aes_ccm_init(struct crypto_tfm
*tfm
)
579 return nx_crypto_ctx_init(crypto_tfm_ctx(tfm
), NX_FC_AES
,
583 int nx_crypto_ctx_aes_gcm_init(struct crypto_tfm
*tfm
)
585 return nx_crypto_ctx_init(crypto_tfm_ctx(tfm
), NX_FC_AES
,
589 int nx_crypto_ctx_aes_ctr_init(struct crypto_tfm
*tfm
)
591 return nx_crypto_ctx_init(crypto_tfm_ctx(tfm
), NX_FC_AES
,
595 int nx_crypto_ctx_aes_cbc_init(struct crypto_tfm
*tfm
)
597 return nx_crypto_ctx_init(crypto_tfm_ctx(tfm
), NX_FC_AES
,
601 int nx_crypto_ctx_aes_ecb_init(struct crypto_tfm
*tfm
)
603 return nx_crypto_ctx_init(crypto_tfm_ctx(tfm
), NX_FC_AES
,
607 int nx_crypto_ctx_sha_init(struct crypto_tfm
*tfm
)
609 return nx_crypto_ctx_init(crypto_tfm_ctx(tfm
), NX_FC_SHA
, NX_MODE_SHA
);
612 int nx_crypto_ctx_aes_xcbc_init(struct crypto_tfm
*tfm
)
614 return nx_crypto_ctx_init(crypto_tfm_ctx(tfm
), NX_FC_AES
,
615 NX_MODE_AES_XCBC_MAC
);
619 * nx_crypto_ctx_exit - destroy a crypto api context
621 * @tfm: the crypto transform pointer for the context
623 * As crypto API contexts are destroyed, this exit hook is called to free the
624 * memory associated with it.
626 void nx_crypto_ctx_exit(struct crypto_tfm
*tfm
)
628 struct nx_crypto_ctx
*nx_ctx
= crypto_tfm_ctx(tfm
);
630 kzfree(nx_ctx
->kmem
);
631 nx_ctx
->csbcpb
= NULL
;
632 nx_ctx
->csbcpb_aead
= NULL
;
633 nx_ctx
->in_sg
= NULL
;
634 nx_ctx
->out_sg
= NULL
;
637 static int nx_probe(struct vio_dev
*viodev
, const struct vio_device_id
*id
)
639 dev_dbg(&viodev
->dev
, "driver probed: %s resource id: 0x%x\n",
640 viodev
->name
, viodev
->resource_id
);
642 if (nx_driver
.viodev
) {
643 dev_err(&viodev
->dev
, "%s: Attempt to register more than one "
644 "instance of the hardware\n", __func__
);
648 nx_driver
.viodev
= viodev
;
650 nx_of_init(&viodev
->dev
, &nx_driver
.of
);
652 return nx_register_algs();
655 static int nx_remove(struct vio_dev
*viodev
)
657 dev_dbg(&viodev
->dev
, "entering nx_remove for UA 0x%x\n",
658 viodev
->unit_address
);
660 if (nx_driver
.of
.status
== NX_OKAY
) {
661 NX_DEBUGFS_FINI(&nx_driver
);
663 crypto_unregister_alg(&nx_ccm_aes_alg
);
664 crypto_unregister_alg(&nx_ccm4309_aes_alg
);
665 crypto_unregister_alg(&nx_gcm_aes_alg
);
666 crypto_unregister_alg(&nx_gcm4106_aes_alg
);
667 crypto_unregister_alg(&nx_ctr_aes_alg
);
668 crypto_unregister_alg(&nx_ctr3686_aes_alg
);
669 crypto_unregister_alg(&nx_cbc_aes_alg
);
670 crypto_unregister_alg(&nx_ecb_aes_alg
);
671 crypto_unregister_shash(&nx_shash_sha256_alg
);
672 crypto_unregister_shash(&nx_shash_sha512_alg
);
673 crypto_unregister_shash(&nx_shash_aes_xcbc_alg
);
680 /* module wide initialization/cleanup */
681 static int __init
nx_init(void)
683 return vio_register_driver(&nx_driver
.viodriver
);
686 static void __exit
nx_fini(void)
688 vio_unregister_driver(&nx_driver
.viodriver
);
691 static struct vio_device_id nx_crypto_driver_ids
[] = {
692 { "ibm,sym-encryption-v1", "ibm,sym-encryption" },
695 MODULE_DEVICE_TABLE(vio
, nx_crypto_driver_ids
);
697 /* driver state structure */
698 struct nx_crypto_driver nx_driver
= {
700 .id_table
= nx_crypto_driver_ids
,
707 module_init(nx_init
);
708 module_exit(nx_fini
);
710 MODULE_AUTHOR("Kent Yoder <yoder1@us.ibm.com>");
711 MODULE_DESCRIPTION(NX_STRING
);
712 MODULE_LICENSE("GPL");
713 MODULE_VERSION(NX_VERSION
);