1 // SPDX-License-Identifier: GPL-2.0-only
2 /* n2_core.c: Niagara2 Stream Processing Unit (SPU) crypto support.
4 * Copyright (C) 2010, 2011 David S. Miller <davem@davemloft.net>
7 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
9 #include <linux/kernel.h>
10 #include <linux/module.h>
12 #include <linux/of_device.h>
13 #include <linux/cpumask.h>
14 #include <linux/slab.h>
15 #include <linux/interrupt.h>
16 #include <linux/crypto.h>
17 #include <crypto/md5.h>
18 #include <crypto/sha.h>
19 #include <crypto/aes.h>
20 #include <crypto/internal/des.h>
21 #include <linux/mutex.h>
22 #include <linux/delay.h>
23 #include <linux/sched.h>
25 #include <crypto/internal/hash.h>
26 #include <crypto/scatterwalk.h>
27 #include <crypto/algapi.h>
29 #include <asm/hypervisor.h>
30 #include <asm/mdesc.h>
34 #define DRV_MODULE_NAME "n2_crypto"
35 #define DRV_MODULE_VERSION "0.2"
36 #define DRV_MODULE_RELDATE "July 28, 2011"
38 static const char version
[] =
39 DRV_MODULE_NAME
".c:v" DRV_MODULE_VERSION
" (" DRV_MODULE_RELDATE
")\n";
41 MODULE_AUTHOR("David S. Miller (davem@davemloft.net)");
42 MODULE_DESCRIPTION("Niagara2 Crypto driver");
43 MODULE_LICENSE("GPL");
44 MODULE_VERSION(DRV_MODULE_VERSION
);
46 #define N2_CRA_PRIORITY 200
48 static DEFINE_MUTEX(spu_lock
);
52 unsigned long qhandle
;
59 struct list_head jobs
;
66 struct list_head list
;
70 struct spu_queue
*queue
;
74 static struct spu_queue
**cpu_to_cwq
;
75 static struct spu_queue
**cpu_to_mau
;
77 static unsigned long spu_next_offset(struct spu_queue
*q
, unsigned long off
)
79 if (q
->q_type
== HV_NCS_QTYPE_MAU
) {
80 off
+= MAU_ENTRY_SIZE
;
81 if (off
== (MAU_ENTRY_SIZE
* MAU_NUM_ENTRIES
))
84 off
+= CWQ_ENTRY_SIZE
;
85 if (off
== (CWQ_ENTRY_SIZE
* CWQ_NUM_ENTRIES
))
91 struct n2_request_common
{
92 struct list_head entry
;
95 #define OFFSET_NOT_RUNNING (~(unsigned int)0)
97 /* An async job request records the final tail value it used in
98 * n2_request_common->offset, test to see if that offset is in
99 * the range old_head, new_head, inclusive.
101 static inline bool job_finished(struct spu_queue
*q
, unsigned int offset
,
102 unsigned long old_head
, unsigned long new_head
)
104 if (old_head
<= new_head
) {
105 if (offset
> old_head
&& offset
<= new_head
)
108 if (offset
> old_head
|| offset
<= new_head
)
114 /* When the HEAD marker is unequal to the actual HEAD, we get
115 * a virtual device INO interrupt. We should process the
116 * completed CWQ entries and adjust the HEAD marker to clear
119 static irqreturn_t
cwq_intr(int irq
, void *dev_id
)
121 unsigned long off
, new_head
, hv_ret
;
122 struct spu_queue
*q
= dev_id
;
124 pr_err("CPU[%d]: Got CWQ interrupt for qhdl[%lx]\n",
125 smp_processor_id(), q
->qhandle
);
129 hv_ret
= sun4v_ncs_gethead(q
->qhandle
, &new_head
);
131 pr_err("CPU[%d]: CWQ gethead[%lx] hv_ret[%lu]\n",
132 smp_processor_id(), new_head
, hv_ret
);
134 for (off
= q
->head
; off
!= new_head
; off
= spu_next_offset(q
, off
)) {
138 hv_ret
= sun4v_ncs_sethead_marker(q
->qhandle
, new_head
);
139 if (hv_ret
== HV_EOK
)
142 spin_unlock(&q
->lock
);
147 static irqreturn_t
mau_intr(int irq
, void *dev_id
)
149 struct spu_queue
*q
= dev_id
;
150 unsigned long head
, hv_ret
;
154 pr_err("CPU[%d]: Got MAU interrupt for qhdl[%lx]\n",
155 smp_processor_id(), q
->qhandle
);
157 hv_ret
= sun4v_ncs_gethead(q
->qhandle
, &head
);
159 pr_err("CPU[%d]: MAU gethead[%lx] hv_ret[%lu]\n",
160 smp_processor_id(), head
, hv_ret
);
162 sun4v_ncs_sethead_marker(q
->qhandle
, head
);
164 spin_unlock(&q
->lock
);
169 static void *spu_queue_next(struct spu_queue
*q
, void *cur
)
171 return q
->q
+ spu_next_offset(q
, cur
- q
->q
);
174 static int spu_queue_num_free(struct spu_queue
*q
)
176 unsigned long head
= q
->head
;
177 unsigned long tail
= q
->tail
;
178 unsigned long end
= (CWQ_ENTRY_SIZE
* CWQ_NUM_ENTRIES
);
184 diff
= (end
- tail
) + head
;
186 return (diff
/ CWQ_ENTRY_SIZE
) - 1;
189 static void *spu_queue_alloc(struct spu_queue
*q
, int num_entries
)
191 int avail
= spu_queue_num_free(q
);
193 if (avail
>= num_entries
)
194 return q
->q
+ q
->tail
;
199 static unsigned long spu_queue_submit(struct spu_queue
*q
, void *last
)
201 unsigned long hv_ret
, new_tail
;
203 new_tail
= spu_next_offset(q
, last
- q
->q
);
205 hv_ret
= sun4v_ncs_settail(q
->qhandle
, new_tail
);
206 if (hv_ret
== HV_EOK
)
211 static u64
control_word_base(unsigned int len
, unsigned int hmac_key_len
,
212 int enc_type
, int auth_type
,
213 unsigned int hash_len
,
214 bool sfas
, bool sob
, bool eob
, bool encrypt
,
217 u64 word
= (len
- 1) & CONTROL_LEN
;
219 word
|= ((u64
) opcode
<< CONTROL_OPCODE_SHIFT
);
220 word
|= ((u64
) enc_type
<< CONTROL_ENC_TYPE_SHIFT
);
221 word
|= ((u64
) auth_type
<< CONTROL_AUTH_TYPE_SHIFT
);
223 word
|= CONTROL_STORE_FINAL_AUTH_STATE
;
225 word
|= CONTROL_START_OF_BLOCK
;
227 word
|= CONTROL_END_OF_BLOCK
;
229 word
|= CONTROL_ENCRYPT
;
231 word
|= ((u64
) (hmac_key_len
- 1)) << CONTROL_HMAC_KEY_LEN_SHIFT
;
233 word
|= ((u64
) (hash_len
- 1)) << CONTROL_HASH_LEN_SHIFT
;
239 static inline bool n2_should_run_async(struct spu_queue
*qp
, int this_len
)
241 if (this_len
>= 64 ||
242 qp
->head
!= qp
->tail
)
248 struct n2_ahash_alg
{
249 struct list_head entry
;
251 const u32
*hash_init
;
256 struct ahash_alg alg
;
259 static inline struct n2_ahash_alg
*n2_ahash_alg(struct crypto_tfm
*tfm
)
261 struct crypto_alg
*alg
= tfm
->__crt_alg
;
262 struct ahash_alg
*ahash_alg
;
264 ahash_alg
= container_of(alg
, struct ahash_alg
, halg
.base
);
266 return container_of(ahash_alg
, struct n2_ahash_alg
, alg
);
270 const char *child_alg
;
271 struct n2_ahash_alg derived
;
274 static inline struct n2_hmac_alg
*n2_hmac_alg(struct crypto_tfm
*tfm
)
276 struct crypto_alg
*alg
= tfm
->__crt_alg
;
277 struct ahash_alg
*ahash_alg
;
279 ahash_alg
= container_of(alg
, struct ahash_alg
, halg
.base
);
281 return container_of(ahash_alg
, struct n2_hmac_alg
, derived
.alg
);
285 struct crypto_ahash
*fallback_tfm
;
288 #define N2_HASH_KEY_MAX 32 /* HW limit for all HMAC requests */
291 struct n2_hash_ctx base
;
293 struct crypto_shash
*child_shash
;
296 unsigned char hash_key
[N2_HASH_KEY_MAX
];
299 struct n2_hash_req_ctx
{
301 struct md5_state md5
;
302 struct sha1_state sha1
;
303 struct sha256_state sha256
;
306 struct ahash_request fallback_req
;
309 static int n2_hash_async_init(struct ahash_request
*req
)
311 struct n2_hash_req_ctx
*rctx
= ahash_request_ctx(req
);
312 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(req
);
313 struct n2_hash_ctx
*ctx
= crypto_ahash_ctx(tfm
);
315 ahash_request_set_tfm(&rctx
->fallback_req
, ctx
->fallback_tfm
);
316 rctx
->fallback_req
.base
.flags
= req
->base
.flags
& CRYPTO_TFM_REQ_MAY_SLEEP
;
318 return crypto_ahash_init(&rctx
->fallback_req
);
321 static int n2_hash_async_update(struct ahash_request
*req
)
323 struct n2_hash_req_ctx
*rctx
= ahash_request_ctx(req
);
324 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(req
);
325 struct n2_hash_ctx
*ctx
= crypto_ahash_ctx(tfm
);
327 ahash_request_set_tfm(&rctx
->fallback_req
, ctx
->fallback_tfm
);
328 rctx
->fallback_req
.base
.flags
= req
->base
.flags
& CRYPTO_TFM_REQ_MAY_SLEEP
;
329 rctx
->fallback_req
.nbytes
= req
->nbytes
;
330 rctx
->fallback_req
.src
= req
->src
;
332 return crypto_ahash_update(&rctx
->fallback_req
);
335 static int n2_hash_async_final(struct ahash_request
*req
)
337 struct n2_hash_req_ctx
*rctx
= ahash_request_ctx(req
);
338 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(req
);
339 struct n2_hash_ctx
*ctx
= crypto_ahash_ctx(tfm
);
341 ahash_request_set_tfm(&rctx
->fallback_req
, ctx
->fallback_tfm
);
342 rctx
->fallback_req
.base
.flags
= req
->base
.flags
& CRYPTO_TFM_REQ_MAY_SLEEP
;
343 rctx
->fallback_req
.result
= req
->result
;
345 return crypto_ahash_final(&rctx
->fallback_req
);
348 static int n2_hash_async_finup(struct ahash_request
*req
)
350 struct n2_hash_req_ctx
*rctx
= ahash_request_ctx(req
);
351 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(req
);
352 struct n2_hash_ctx
*ctx
= crypto_ahash_ctx(tfm
);
354 ahash_request_set_tfm(&rctx
->fallback_req
, ctx
->fallback_tfm
);
355 rctx
->fallback_req
.base
.flags
= req
->base
.flags
& CRYPTO_TFM_REQ_MAY_SLEEP
;
356 rctx
->fallback_req
.nbytes
= req
->nbytes
;
357 rctx
->fallback_req
.src
= req
->src
;
358 rctx
->fallback_req
.result
= req
->result
;
360 return crypto_ahash_finup(&rctx
->fallback_req
);
363 static int n2_hash_async_noimport(struct ahash_request
*req
, const void *in
)
368 static int n2_hash_async_noexport(struct ahash_request
*req
, void *out
)
373 static int n2_hash_cra_init(struct crypto_tfm
*tfm
)
375 const char *fallback_driver_name
= crypto_tfm_alg_name(tfm
);
376 struct crypto_ahash
*ahash
= __crypto_ahash_cast(tfm
);
377 struct n2_hash_ctx
*ctx
= crypto_ahash_ctx(ahash
);
378 struct crypto_ahash
*fallback_tfm
;
381 fallback_tfm
= crypto_alloc_ahash(fallback_driver_name
, 0,
382 CRYPTO_ALG_NEED_FALLBACK
);
383 if (IS_ERR(fallback_tfm
)) {
384 pr_warning("Fallback driver '%s' could not be loaded!\n",
385 fallback_driver_name
);
386 err
= PTR_ERR(fallback_tfm
);
390 crypto_ahash_set_reqsize(ahash
, (sizeof(struct n2_hash_req_ctx
) +
391 crypto_ahash_reqsize(fallback_tfm
)));
393 ctx
->fallback_tfm
= fallback_tfm
;
400 static void n2_hash_cra_exit(struct crypto_tfm
*tfm
)
402 struct crypto_ahash
*ahash
= __crypto_ahash_cast(tfm
);
403 struct n2_hash_ctx
*ctx
= crypto_ahash_ctx(ahash
);
405 crypto_free_ahash(ctx
->fallback_tfm
);
408 static int n2_hmac_cra_init(struct crypto_tfm
*tfm
)
410 const char *fallback_driver_name
= crypto_tfm_alg_name(tfm
);
411 struct crypto_ahash
*ahash
= __crypto_ahash_cast(tfm
);
412 struct n2_hmac_ctx
*ctx
= crypto_ahash_ctx(ahash
);
413 struct n2_hmac_alg
*n2alg
= n2_hmac_alg(tfm
);
414 struct crypto_ahash
*fallback_tfm
;
415 struct crypto_shash
*child_shash
;
418 fallback_tfm
= crypto_alloc_ahash(fallback_driver_name
, 0,
419 CRYPTO_ALG_NEED_FALLBACK
);
420 if (IS_ERR(fallback_tfm
)) {
421 pr_warning("Fallback driver '%s' could not be loaded!\n",
422 fallback_driver_name
);
423 err
= PTR_ERR(fallback_tfm
);
427 child_shash
= crypto_alloc_shash(n2alg
->child_alg
, 0, 0);
428 if (IS_ERR(child_shash
)) {
429 pr_warning("Child shash '%s' could not be loaded!\n",
431 err
= PTR_ERR(child_shash
);
432 goto out_free_fallback
;
435 crypto_ahash_set_reqsize(ahash
, (sizeof(struct n2_hash_req_ctx
) +
436 crypto_ahash_reqsize(fallback_tfm
)));
438 ctx
->child_shash
= child_shash
;
439 ctx
->base
.fallback_tfm
= fallback_tfm
;
443 crypto_free_ahash(fallback_tfm
);
449 static void n2_hmac_cra_exit(struct crypto_tfm
*tfm
)
451 struct crypto_ahash
*ahash
= __crypto_ahash_cast(tfm
);
452 struct n2_hmac_ctx
*ctx
= crypto_ahash_ctx(ahash
);
454 crypto_free_ahash(ctx
->base
.fallback_tfm
);
455 crypto_free_shash(ctx
->child_shash
);
458 static int n2_hmac_async_setkey(struct crypto_ahash
*tfm
, const u8
*key
,
461 struct n2_hmac_ctx
*ctx
= crypto_ahash_ctx(tfm
);
462 struct crypto_shash
*child_shash
= ctx
->child_shash
;
463 struct crypto_ahash
*fallback_tfm
;
464 SHASH_DESC_ON_STACK(shash
, child_shash
);
467 fallback_tfm
= ctx
->base
.fallback_tfm
;
468 err
= crypto_ahash_setkey(fallback_tfm
, key
, keylen
);
472 shash
->tfm
= child_shash
;
474 bs
= crypto_shash_blocksize(child_shash
);
475 ds
= crypto_shash_digestsize(child_shash
);
476 BUG_ON(ds
> N2_HASH_KEY_MAX
);
478 err
= crypto_shash_digest(shash
, key
, keylen
,
483 } else if (keylen
<= N2_HASH_KEY_MAX
)
484 memcpy(ctx
->hash_key
, key
, keylen
);
486 ctx
->hash_key_len
= keylen
;
491 static unsigned long wait_for_tail(struct spu_queue
*qp
)
493 unsigned long head
, hv_ret
;
496 hv_ret
= sun4v_ncs_gethead(qp
->qhandle
, &head
);
497 if (hv_ret
!= HV_EOK
) {
498 pr_err("Hypervisor error on gethead\n");
501 if (head
== qp
->tail
) {
509 static unsigned long submit_and_wait_for_tail(struct spu_queue
*qp
,
510 struct cwq_initial_entry
*ent
)
512 unsigned long hv_ret
= spu_queue_submit(qp
, ent
);
514 if (hv_ret
== HV_EOK
)
515 hv_ret
= wait_for_tail(qp
);
520 static int n2_do_async_digest(struct ahash_request
*req
,
521 unsigned int auth_type
, unsigned int digest_size
,
522 unsigned int result_size
, void *hash_loc
,
523 unsigned long auth_key
, unsigned int auth_key_len
)
525 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(req
);
526 struct cwq_initial_entry
*ent
;
527 struct crypto_hash_walk walk
;
528 struct spu_queue
*qp
;
533 /* The total effective length of the operation may not
536 if (unlikely(req
->nbytes
> (1 << 16))) {
537 struct n2_hash_req_ctx
*rctx
= ahash_request_ctx(req
);
538 struct n2_hash_ctx
*ctx
= crypto_ahash_ctx(tfm
);
540 ahash_request_set_tfm(&rctx
->fallback_req
, ctx
->fallback_tfm
);
541 rctx
->fallback_req
.base
.flags
=
542 req
->base
.flags
& CRYPTO_TFM_REQ_MAY_SLEEP
;
543 rctx
->fallback_req
.nbytes
= req
->nbytes
;
544 rctx
->fallback_req
.src
= req
->src
;
545 rctx
->fallback_req
.result
= req
->result
;
547 return crypto_ahash_digest(&rctx
->fallback_req
);
550 nbytes
= crypto_hash_walk_first(req
, &walk
);
553 qp
= cpu_to_cwq
[cpu
];
557 spin_lock_irqsave(&qp
->lock
, flags
);
559 /* XXX can do better, improve this later by doing a by-hand scatterlist
562 ent
= qp
->q
+ qp
->tail
;
564 ent
->control
= control_word_base(nbytes
, auth_key_len
, 0,
565 auth_type
, digest_size
,
566 false, true, false, false,
569 ent
->src_addr
= __pa(walk
.data
);
570 ent
->auth_key_addr
= auth_key
;
571 ent
->auth_iv_addr
= __pa(hash_loc
);
572 ent
->final_auth_state_addr
= 0UL;
573 ent
->enc_key_addr
= 0UL;
574 ent
->enc_iv_addr
= 0UL;
575 ent
->dest_addr
= __pa(hash_loc
);
577 nbytes
= crypto_hash_walk_done(&walk
, 0);
579 ent
= spu_queue_next(qp
, ent
);
581 ent
->control
= (nbytes
- 1);
582 ent
->src_addr
= __pa(walk
.data
);
583 ent
->auth_key_addr
= 0UL;
584 ent
->auth_iv_addr
= 0UL;
585 ent
->final_auth_state_addr
= 0UL;
586 ent
->enc_key_addr
= 0UL;
587 ent
->enc_iv_addr
= 0UL;
588 ent
->dest_addr
= 0UL;
590 nbytes
= crypto_hash_walk_done(&walk
, 0);
592 ent
->control
|= CONTROL_END_OF_BLOCK
;
594 if (submit_and_wait_for_tail(qp
, ent
) != HV_EOK
)
599 spin_unlock_irqrestore(&qp
->lock
, flags
);
602 memcpy(req
->result
, hash_loc
, result_size
);
609 static int n2_hash_async_digest(struct ahash_request
*req
)
611 struct n2_ahash_alg
*n2alg
= n2_ahash_alg(req
->base
.tfm
);
612 struct n2_hash_req_ctx
*rctx
= ahash_request_ctx(req
);
615 ds
= n2alg
->digest_size
;
616 if (unlikely(req
->nbytes
== 0)) {
617 memcpy(req
->result
, n2alg
->hash_zero
, ds
);
620 memcpy(&rctx
->u
, n2alg
->hash_init
, n2alg
->hw_op_hashsz
);
622 return n2_do_async_digest(req
, n2alg
->auth_type
,
623 n2alg
->hw_op_hashsz
, ds
,
627 static int n2_hmac_async_digest(struct ahash_request
*req
)
629 struct n2_hmac_alg
*n2alg
= n2_hmac_alg(req
->base
.tfm
);
630 struct n2_hash_req_ctx
*rctx
= ahash_request_ctx(req
);
631 struct crypto_ahash
*tfm
= crypto_ahash_reqtfm(req
);
632 struct n2_hmac_ctx
*ctx
= crypto_ahash_ctx(tfm
);
635 ds
= n2alg
->derived
.digest_size
;
636 if (unlikely(req
->nbytes
== 0) ||
637 unlikely(ctx
->hash_key_len
> N2_HASH_KEY_MAX
)) {
638 struct n2_hash_req_ctx
*rctx
= ahash_request_ctx(req
);
639 struct n2_hash_ctx
*ctx
= crypto_ahash_ctx(tfm
);
641 ahash_request_set_tfm(&rctx
->fallback_req
, ctx
->fallback_tfm
);
642 rctx
->fallback_req
.base
.flags
=
643 req
->base
.flags
& CRYPTO_TFM_REQ_MAY_SLEEP
;
644 rctx
->fallback_req
.nbytes
= req
->nbytes
;
645 rctx
->fallback_req
.src
= req
->src
;
646 rctx
->fallback_req
.result
= req
->result
;
648 return crypto_ahash_digest(&rctx
->fallback_req
);
650 memcpy(&rctx
->u
, n2alg
->derived
.hash_init
,
651 n2alg
->derived
.hw_op_hashsz
);
653 return n2_do_async_digest(req
, n2alg
->derived
.hmac_type
,
654 n2alg
->derived
.hw_op_hashsz
, ds
,
656 __pa(&ctx
->hash_key
),
660 struct n2_cipher_context
{
664 u8 aes
[AES_MAX_KEY_SIZE
];
665 u8 des
[DES_KEY_SIZE
];
666 u8 des3
[3 * DES_KEY_SIZE
];
667 u8 arc4
[258]; /* S-box, X, Y */
671 #define N2_CHUNK_ARR_LEN 16
673 struct n2_crypto_chunk
{
674 struct list_head entry
;
675 unsigned long iv_paddr
: 44;
676 unsigned long arr_len
: 20;
677 unsigned long dest_paddr
;
678 unsigned long dest_final
;
680 unsigned long src_paddr
: 44;
681 unsigned long src_len
: 20;
682 } arr
[N2_CHUNK_ARR_LEN
];
685 struct n2_request_context
{
686 struct ablkcipher_walk walk
;
687 struct list_head chunk_list
;
688 struct n2_crypto_chunk chunk
;
692 /* The SPU allows some level of flexibility for partial cipher blocks
693 * being specified in a descriptor.
695 * It merely requires that every descriptor's length field is at least
696 * as large as the cipher block size. This means that a cipher block
697 * can span at most 2 descriptors. However, this does not allow a
698 * partial block to span into the final descriptor as that would
699 * violate the rule (since every descriptor's length must be at lest
700 * the block size). So, for example, assuming an 8 byte block size:
702 * 0xe --> 0xa --> 0x8
704 * is a valid length sequence, whereas:
706 * 0xe --> 0xb --> 0x7
708 * is not a valid sequence.
711 struct n2_cipher_alg
{
712 struct list_head entry
;
714 struct crypto_alg alg
;
717 static inline struct n2_cipher_alg
*n2_cipher_alg(struct crypto_tfm
*tfm
)
719 struct crypto_alg
*alg
= tfm
->__crt_alg
;
721 return container_of(alg
, struct n2_cipher_alg
, alg
);
724 struct n2_cipher_request_context
{
725 struct ablkcipher_walk walk
;
728 static int n2_aes_setkey(struct crypto_ablkcipher
*cipher
, const u8
*key
,
731 struct crypto_tfm
*tfm
= crypto_ablkcipher_tfm(cipher
);
732 struct n2_cipher_context
*ctx
= crypto_tfm_ctx(tfm
);
733 struct n2_cipher_alg
*n2alg
= n2_cipher_alg(tfm
);
735 ctx
->enc_type
= (n2alg
->enc_type
& ENC_TYPE_CHAINING_MASK
);
738 case AES_KEYSIZE_128
:
739 ctx
->enc_type
|= ENC_TYPE_ALG_AES128
;
741 case AES_KEYSIZE_192
:
742 ctx
->enc_type
|= ENC_TYPE_ALG_AES192
;
744 case AES_KEYSIZE_256
:
745 ctx
->enc_type
|= ENC_TYPE_ALG_AES256
;
748 crypto_ablkcipher_set_flags(cipher
, CRYPTO_TFM_RES_BAD_KEY_LEN
);
752 ctx
->key_len
= keylen
;
753 memcpy(ctx
->key
.aes
, key
, keylen
);
757 static int n2_des_setkey(struct crypto_ablkcipher
*cipher
, const u8
*key
,
760 struct crypto_tfm
*tfm
= crypto_ablkcipher_tfm(cipher
);
761 struct n2_cipher_context
*ctx
= crypto_tfm_ctx(tfm
);
762 struct n2_cipher_alg
*n2alg
= n2_cipher_alg(tfm
);
765 err
= verify_ablkcipher_des_key(cipher
, key
);
769 ctx
->enc_type
= n2alg
->enc_type
;
771 ctx
->key_len
= keylen
;
772 memcpy(ctx
->key
.des
, key
, keylen
);
776 static int n2_3des_setkey(struct crypto_ablkcipher
*cipher
, const u8
*key
,
779 struct crypto_tfm
*tfm
= crypto_ablkcipher_tfm(cipher
);
780 struct n2_cipher_context
*ctx
= crypto_tfm_ctx(tfm
);
781 struct n2_cipher_alg
*n2alg
= n2_cipher_alg(tfm
);
784 err
= verify_ablkcipher_des3_key(cipher
, key
);
788 ctx
->enc_type
= n2alg
->enc_type
;
790 ctx
->key_len
= keylen
;
791 memcpy(ctx
->key
.des3
, key
, keylen
);
795 static int n2_arc4_setkey(struct crypto_ablkcipher
*cipher
, const u8
*key
,
798 struct crypto_tfm
*tfm
= crypto_ablkcipher_tfm(cipher
);
799 struct n2_cipher_context
*ctx
= crypto_tfm_ctx(tfm
);
800 struct n2_cipher_alg
*n2alg
= n2_cipher_alg(tfm
);
801 u8
*s
= ctx
->key
.arc4
;
806 ctx
->enc_type
= n2alg
->enc_type
;
811 for (i
= 0; i
< 256; i
++)
813 for (i
= 0; i
< 256; i
++) {
815 j
= (j
+ key
[k
] + a
) & 0xff;
825 static inline int cipher_descriptor_len(int nbytes
, unsigned int block_size
)
827 int this_len
= nbytes
;
829 this_len
-= (nbytes
& (block_size
- 1));
830 return this_len
> (1 << 16) ? (1 << 16) : this_len
;
833 static int __n2_crypt_chunk(struct crypto_tfm
*tfm
, struct n2_crypto_chunk
*cp
,
834 struct spu_queue
*qp
, bool encrypt
)
836 struct n2_cipher_context
*ctx
= crypto_tfm_ctx(tfm
);
837 struct cwq_initial_entry
*ent
;
841 ent
= spu_queue_alloc(qp
, cp
->arr_len
);
843 pr_info("queue_alloc() of %d fails\n",
848 in_place
= (cp
->dest_paddr
== cp
->arr
[0].src_paddr
);
850 ent
->control
= control_word_base(cp
->arr
[0].src_len
,
851 0, ctx
->enc_type
, 0, 0,
852 false, true, false, encrypt
,
854 (in_place
? OPCODE_INPLACE_BIT
: 0));
855 ent
->src_addr
= cp
->arr
[0].src_paddr
;
856 ent
->auth_key_addr
= 0UL;
857 ent
->auth_iv_addr
= 0UL;
858 ent
->final_auth_state_addr
= 0UL;
859 ent
->enc_key_addr
= __pa(&ctx
->key
);
860 ent
->enc_iv_addr
= cp
->iv_paddr
;
861 ent
->dest_addr
= (in_place
? 0UL : cp
->dest_paddr
);
863 for (i
= 1; i
< cp
->arr_len
; i
++) {
864 ent
= spu_queue_next(qp
, ent
);
866 ent
->control
= cp
->arr
[i
].src_len
- 1;
867 ent
->src_addr
= cp
->arr
[i
].src_paddr
;
868 ent
->auth_key_addr
= 0UL;
869 ent
->auth_iv_addr
= 0UL;
870 ent
->final_auth_state_addr
= 0UL;
871 ent
->enc_key_addr
= 0UL;
872 ent
->enc_iv_addr
= 0UL;
873 ent
->dest_addr
= 0UL;
875 ent
->control
|= CONTROL_END_OF_BLOCK
;
877 return (spu_queue_submit(qp
, ent
) != HV_EOK
) ? -EINVAL
: 0;
880 static int n2_compute_chunks(struct ablkcipher_request
*req
)
882 struct n2_request_context
*rctx
= ablkcipher_request_ctx(req
);
883 struct ablkcipher_walk
*walk
= &rctx
->walk
;
884 struct n2_crypto_chunk
*chunk
;
885 unsigned long dest_prev
;
886 unsigned int tot_len
;
890 ablkcipher_walk_init(walk
, req
->dst
, req
->src
, req
->nbytes
);
891 err
= ablkcipher_walk_phys(req
, walk
);
895 INIT_LIST_HEAD(&rctx
->chunk_list
);
897 chunk
= &rctx
->chunk
;
898 INIT_LIST_HEAD(&chunk
->entry
);
900 chunk
->iv_paddr
= 0UL;
902 chunk
->dest_paddr
= 0UL;
904 prev_in_place
= false;
908 while ((nbytes
= walk
->nbytes
) != 0) {
909 unsigned long dest_paddr
, src_paddr
;
913 src_paddr
= (page_to_phys(walk
->src
.page
) +
915 dest_paddr
= (page_to_phys(walk
->dst
.page
) +
917 in_place
= (src_paddr
== dest_paddr
);
918 this_len
= cipher_descriptor_len(nbytes
, walk
->blocksize
);
920 if (chunk
->arr_len
!= 0) {
921 if (in_place
!= prev_in_place
||
923 dest_paddr
!= dest_prev
) ||
924 chunk
->arr_len
== N2_CHUNK_ARR_LEN
||
925 tot_len
+ this_len
> (1 << 16)) {
926 chunk
->dest_final
= dest_prev
;
927 list_add_tail(&chunk
->entry
,
929 chunk
= kzalloc(sizeof(*chunk
), GFP_ATOMIC
);
934 INIT_LIST_HEAD(&chunk
->entry
);
937 if (chunk
->arr_len
== 0) {
938 chunk
->dest_paddr
= dest_paddr
;
941 chunk
->arr
[chunk
->arr_len
].src_paddr
= src_paddr
;
942 chunk
->arr
[chunk
->arr_len
].src_len
= this_len
;
945 dest_prev
= dest_paddr
+ this_len
;
946 prev_in_place
= in_place
;
949 err
= ablkcipher_walk_done(req
, walk
, nbytes
- this_len
);
953 if (!err
&& chunk
->arr_len
!= 0) {
954 chunk
->dest_final
= dest_prev
;
955 list_add_tail(&chunk
->entry
, &rctx
->chunk_list
);
961 static void n2_chunk_complete(struct ablkcipher_request
*req
, void *final_iv
)
963 struct n2_request_context
*rctx
= ablkcipher_request_ctx(req
);
964 struct n2_crypto_chunk
*c
, *tmp
;
967 memcpy(rctx
->walk
.iv
, final_iv
, rctx
->walk
.blocksize
);
969 ablkcipher_walk_complete(&rctx
->walk
);
970 list_for_each_entry_safe(c
, tmp
, &rctx
->chunk_list
, entry
) {
972 if (unlikely(c
!= &rctx
->chunk
))
978 static int n2_do_ecb(struct ablkcipher_request
*req
, bool encrypt
)
980 struct n2_request_context
*rctx
= ablkcipher_request_ctx(req
);
981 struct crypto_tfm
*tfm
= req
->base
.tfm
;
982 int err
= n2_compute_chunks(req
);
983 struct n2_crypto_chunk
*c
, *tmp
;
984 unsigned long flags
, hv_ret
;
985 struct spu_queue
*qp
;
990 qp
= cpu_to_cwq
[get_cpu()];
995 spin_lock_irqsave(&qp
->lock
, flags
);
997 list_for_each_entry_safe(c
, tmp
, &rctx
->chunk_list
, entry
) {
998 err
= __n2_crypt_chunk(tfm
, c
, qp
, encrypt
);
1001 list_del(&c
->entry
);
1002 if (unlikely(c
!= &rctx
->chunk
))
1006 hv_ret
= wait_for_tail(qp
);
1007 if (hv_ret
!= HV_EOK
)
1011 spin_unlock_irqrestore(&qp
->lock
, flags
);
1016 n2_chunk_complete(req
, NULL
);
1020 static int n2_encrypt_ecb(struct ablkcipher_request
*req
)
1022 return n2_do_ecb(req
, true);
1025 static int n2_decrypt_ecb(struct ablkcipher_request
*req
)
1027 return n2_do_ecb(req
, false);
1030 static int n2_do_chaining(struct ablkcipher_request
*req
, bool encrypt
)
1032 struct n2_request_context
*rctx
= ablkcipher_request_ctx(req
);
1033 struct crypto_tfm
*tfm
= req
->base
.tfm
;
1034 unsigned long flags
, hv_ret
, iv_paddr
;
1035 int err
= n2_compute_chunks(req
);
1036 struct n2_crypto_chunk
*c
, *tmp
;
1037 struct spu_queue
*qp
;
1038 void *final_iv_addr
;
1040 final_iv_addr
= NULL
;
1045 qp
= cpu_to_cwq
[get_cpu()];
1050 spin_lock_irqsave(&qp
->lock
, flags
);
1053 iv_paddr
= __pa(rctx
->walk
.iv
);
1054 list_for_each_entry_safe(c
, tmp
, &rctx
->chunk_list
,
1056 c
->iv_paddr
= iv_paddr
;
1057 err
= __n2_crypt_chunk(tfm
, c
, qp
, true);
1060 iv_paddr
= c
->dest_final
- rctx
->walk
.blocksize
;
1061 list_del(&c
->entry
);
1062 if (unlikely(c
!= &rctx
->chunk
))
1065 final_iv_addr
= __va(iv_paddr
);
1067 list_for_each_entry_safe_reverse(c
, tmp
, &rctx
->chunk_list
,
1069 if (c
== &rctx
->chunk
) {
1070 iv_paddr
= __pa(rctx
->walk
.iv
);
1072 iv_paddr
= (tmp
->arr
[tmp
->arr_len
-1].src_paddr
+
1073 tmp
->arr
[tmp
->arr_len
-1].src_len
-
1074 rctx
->walk
.blocksize
);
1076 if (!final_iv_addr
) {
1079 pa
= (c
->arr
[c
->arr_len
-1].src_paddr
+
1080 c
->arr
[c
->arr_len
-1].src_len
-
1081 rctx
->walk
.blocksize
);
1082 final_iv_addr
= rctx
->temp_iv
;
1083 memcpy(rctx
->temp_iv
, __va(pa
),
1084 rctx
->walk
.blocksize
);
1086 c
->iv_paddr
= iv_paddr
;
1087 err
= __n2_crypt_chunk(tfm
, c
, qp
, false);
1090 list_del(&c
->entry
);
1091 if (unlikely(c
!= &rctx
->chunk
))
1096 hv_ret
= wait_for_tail(qp
);
1097 if (hv_ret
!= HV_EOK
)
1101 spin_unlock_irqrestore(&qp
->lock
, flags
);
1106 n2_chunk_complete(req
, err
? NULL
: final_iv_addr
);
1110 static int n2_encrypt_chaining(struct ablkcipher_request
*req
)
1112 return n2_do_chaining(req
, true);
1115 static int n2_decrypt_chaining(struct ablkcipher_request
*req
)
1117 return n2_do_chaining(req
, false);
1120 struct n2_cipher_tmpl
{
1122 const char *drv_name
;
1125 struct ablkcipher_alg ablkcipher
;
1128 static const struct n2_cipher_tmpl cipher_tmpls
[] = {
1129 /* ARC4: only ECB is supported (chaining bits ignored) */
1130 { .name
= "ecb(arc4)",
1131 .drv_name
= "ecb-arc4",
1133 .enc_type
= (ENC_TYPE_ALG_RC4_STREAM
|
1134 ENC_TYPE_CHAINING_ECB
),
1138 .setkey
= n2_arc4_setkey
,
1139 .encrypt
= n2_encrypt_ecb
,
1140 .decrypt
= n2_decrypt_ecb
,
1144 /* DES: ECB CBC and CFB are supported */
1145 { .name
= "ecb(des)",
1146 .drv_name
= "ecb-des",
1147 .block_size
= DES_BLOCK_SIZE
,
1148 .enc_type
= (ENC_TYPE_ALG_DES
|
1149 ENC_TYPE_CHAINING_ECB
),
1151 .min_keysize
= DES_KEY_SIZE
,
1152 .max_keysize
= DES_KEY_SIZE
,
1153 .setkey
= n2_des_setkey
,
1154 .encrypt
= n2_encrypt_ecb
,
1155 .decrypt
= n2_decrypt_ecb
,
1158 { .name
= "cbc(des)",
1159 .drv_name
= "cbc-des",
1160 .block_size
= DES_BLOCK_SIZE
,
1161 .enc_type
= (ENC_TYPE_ALG_DES
|
1162 ENC_TYPE_CHAINING_CBC
),
1164 .ivsize
= DES_BLOCK_SIZE
,
1165 .min_keysize
= DES_KEY_SIZE
,
1166 .max_keysize
= DES_KEY_SIZE
,
1167 .setkey
= n2_des_setkey
,
1168 .encrypt
= n2_encrypt_chaining
,
1169 .decrypt
= n2_decrypt_chaining
,
1172 { .name
= "cfb(des)",
1173 .drv_name
= "cfb-des",
1174 .block_size
= DES_BLOCK_SIZE
,
1175 .enc_type
= (ENC_TYPE_ALG_DES
|
1176 ENC_TYPE_CHAINING_CFB
),
1178 .min_keysize
= DES_KEY_SIZE
,
1179 .max_keysize
= DES_KEY_SIZE
,
1180 .setkey
= n2_des_setkey
,
1181 .encrypt
= n2_encrypt_chaining
,
1182 .decrypt
= n2_decrypt_chaining
,
1186 /* 3DES: ECB CBC and CFB are supported */
1187 { .name
= "ecb(des3_ede)",
1188 .drv_name
= "ecb-3des",
1189 .block_size
= DES_BLOCK_SIZE
,
1190 .enc_type
= (ENC_TYPE_ALG_3DES
|
1191 ENC_TYPE_CHAINING_ECB
),
1193 .min_keysize
= 3 * DES_KEY_SIZE
,
1194 .max_keysize
= 3 * DES_KEY_SIZE
,
1195 .setkey
= n2_3des_setkey
,
1196 .encrypt
= n2_encrypt_ecb
,
1197 .decrypt
= n2_decrypt_ecb
,
1200 { .name
= "cbc(des3_ede)",
1201 .drv_name
= "cbc-3des",
1202 .block_size
= DES_BLOCK_SIZE
,
1203 .enc_type
= (ENC_TYPE_ALG_3DES
|
1204 ENC_TYPE_CHAINING_CBC
),
1206 .ivsize
= DES_BLOCK_SIZE
,
1207 .min_keysize
= 3 * DES_KEY_SIZE
,
1208 .max_keysize
= 3 * DES_KEY_SIZE
,
1209 .setkey
= n2_3des_setkey
,
1210 .encrypt
= n2_encrypt_chaining
,
1211 .decrypt
= n2_decrypt_chaining
,
1214 { .name
= "cfb(des3_ede)",
1215 .drv_name
= "cfb-3des",
1216 .block_size
= DES_BLOCK_SIZE
,
1217 .enc_type
= (ENC_TYPE_ALG_3DES
|
1218 ENC_TYPE_CHAINING_CFB
),
1220 .min_keysize
= 3 * DES_KEY_SIZE
,
1221 .max_keysize
= 3 * DES_KEY_SIZE
,
1222 .setkey
= n2_3des_setkey
,
1223 .encrypt
= n2_encrypt_chaining
,
1224 .decrypt
= n2_decrypt_chaining
,
1227 /* AES: ECB CBC and CTR are supported */
1228 { .name
= "ecb(aes)",
1229 .drv_name
= "ecb-aes",
1230 .block_size
= AES_BLOCK_SIZE
,
1231 .enc_type
= (ENC_TYPE_ALG_AES128
|
1232 ENC_TYPE_CHAINING_ECB
),
1234 .min_keysize
= AES_MIN_KEY_SIZE
,
1235 .max_keysize
= AES_MAX_KEY_SIZE
,
1236 .setkey
= n2_aes_setkey
,
1237 .encrypt
= n2_encrypt_ecb
,
1238 .decrypt
= n2_decrypt_ecb
,
1241 { .name
= "cbc(aes)",
1242 .drv_name
= "cbc-aes",
1243 .block_size
= AES_BLOCK_SIZE
,
1244 .enc_type
= (ENC_TYPE_ALG_AES128
|
1245 ENC_TYPE_CHAINING_CBC
),
1247 .ivsize
= AES_BLOCK_SIZE
,
1248 .min_keysize
= AES_MIN_KEY_SIZE
,
1249 .max_keysize
= AES_MAX_KEY_SIZE
,
1250 .setkey
= n2_aes_setkey
,
1251 .encrypt
= n2_encrypt_chaining
,
1252 .decrypt
= n2_decrypt_chaining
,
1255 { .name
= "ctr(aes)",
1256 .drv_name
= "ctr-aes",
1257 .block_size
= AES_BLOCK_SIZE
,
1258 .enc_type
= (ENC_TYPE_ALG_AES128
|
1259 ENC_TYPE_CHAINING_COUNTER
),
1261 .ivsize
= AES_BLOCK_SIZE
,
1262 .min_keysize
= AES_MIN_KEY_SIZE
,
1263 .max_keysize
= AES_MAX_KEY_SIZE
,
1264 .setkey
= n2_aes_setkey
,
1265 .encrypt
= n2_encrypt_chaining
,
1266 .decrypt
= n2_encrypt_chaining
,
1271 #define NUM_CIPHER_TMPLS ARRAY_SIZE(cipher_tmpls)
1273 static LIST_HEAD(cipher_algs
);
1275 struct n2_hash_tmpl
{
1277 const u8
*hash_zero
;
1278 const u32
*hash_init
;
1286 static const u32 n2_md5_init
[MD5_HASH_WORDS
] = {
1287 cpu_to_le32(MD5_H0
),
1288 cpu_to_le32(MD5_H1
),
1289 cpu_to_le32(MD5_H2
),
1290 cpu_to_le32(MD5_H3
),
1292 static const u32 n2_sha1_init
[SHA1_DIGEST_SIZE
/ 4] = {
1293 SHA1_H0
, SHA1_H1
, SHA1_H2
, SHA1_H3
, SHA1_H4
,
1295 static const u32 n2_sha256_init
[SHA256_DIGEST_SIZE
/ 4] = {
1296 SHA256_H0
, SHA256_H1
, SHA256_H2
, SHA256_H3
,
1297 SHA256_H4
, SHA256_H5
, SHA256_H6
, SHA256_H7
,
1299 static const u32 n2_sha224_init
[SHA256_DIGEST_SIZE
/ 4] = {
1300 SHA224_H0
, SHA224_H1
, SHA224_H2
, SHA224_H3
,
1301 SHA224_H4
, SHA224_H5
, SHA224_H6
, SHA224_H7
,
1304 static const struct n2_hash_tmpl hash_tmpls
[] = {
1306 .hash_zero
= md5_zero_message_hash
,
1307 .hash_init
= n2_md5_init
,
1308 .auth_type
= AUTH_TYPE_MD5
,
1309 .hmac_type
= AUTH_TYPE_HMAC_MD5
,
1310 .hw_op_hashsz
= MD5_DIGEST_SIZE
,
1311 .digest_size
= MD5_DIGEST_SIZE
,
1312 .block_size
= MD5_HMAC_BLOCK_SIZE
},
1314 .hash_zero
= sha1_zero_message_hash
,
1315 .hash_init
= n2_sha1_init
,
1316 .auth_type
= AUTH_TYPE_SHA1
,
1317 .hmac_type
= AUTH_TYPE_HMAC_SHA1
,
1318 .hw_op_hashsz
= SHA1_DIGEST_SIZE
,
1319 .digest_size
= SHA1_DIGEST_SIZE
,
1320 .block_size
= SHA1_BLOCK_SIZE
},
1322 .hash_zero
= sha256_zero_message_hash
,
1323 .hash_init
= n2_sha256_init
,
1324 .auth_type
= AUTH_TYPE_SHA256
,
1325 .hmac_type
= AUTH_TYPE_HMAC_SHA256
,
1326 .hw_op_hashsz
= SHA256_DIGEST_SIZE
,
1327 .digest_size
= SHA256_DIGEST_SIZE
,
1328 .block_size
= SHA256_BLOCK_SIZE
},
1330 .hash_zero
= sha224_zero_message_hash
,
1331 .hash_init
= n2_sha224_init
,
1332 .auth_type
= AUTH_TYPE_SHA256
,
1333 .hmac_type
= AUTH_TYPE_RESERVED
,
1334 .hw_op_hashsz
= SHA256_DIGEST_SIZE
,
1335 .digest_size
= SHA224_DIGEST_SIZE
,
1336 .block_size
= SHA224_BLOCK_SIZE
},
1338 #define NUM_HASH_TMPLS ARRAY_SIZE(hash_tmpls)
1340 static LIST_HEAD(ahash_algs
);
1341 static LIST_HEAD(hmac_algs
);
1343 static int algs_registered
;
1345 static void __n2_unregister_algs(void)
1347 struct n2_cipher_alg
*cipher
, *cipher_tmp
;
1348 struct n2_ahash_alg
*alg
, *alg_tmp
;
1349 struct n2_hmac_alg
*hmac
, *hmac_tmp
;
1351 list_for_each_entry_safe(cipher
, cipher_tmp
, &cipher_algs
, entry
) {
1352 crypto_unregister_alg(&cipher
->alg
);
1353 list_del(&cipher
->entry
);
1356 list_for_each_entry_safe(hmac
, hmac_tmp
, &hmac_algs
, derived
.entry
) {
1357 crypto_unregister_ahash(&hmac
->derived
.alg
);
1358 list_del(&hmac
->derived
.entry
);
1361 list_for_each_entry_safe(alg
, alg_tmp
, &ahash_algs
, entry
) {
1362 crypto_unregister_ahash(&alg
->alg
);
1363 list_del(&alg
->entry
);
1368 static int n2_cipher_cra_init(struct crypto_tfm
*tfm
)
1370 tfm
->crt_ablkcipher
.reqsize
= sizeof(struct n2_request_context
);
1374 static int __n2_register_one_cipher(const struct n2_cipher_tmpl
*tmpl
)
1376 struct n2_cipher_alg
*p
= kzalloc(sizeof(*p
), GFP_KERNEL
);
1377 struct crypto_alg
*alg
;
1385 snprintf(alg
->cra_name
, CRYPTO_MAX_ALG_NAME
, "%s", tmpl
->name
);
1386 snprintf(alg
->cra_driver_name
, CRYPTO_MAX_ALG_NAME
, "%s-n2", tmpl
->drv_name
);
1387 alg
->cra_priority
= N2_CRA_PRIORITY
;
1388 alg
->cra_flags
= CRYPTO_ALG_TYPE_ABLKCIPHER
|
1389 CRYPTO_ALG_KERN_DRIVER_ONLY
| CRYPTO_ALG_ASYNC
;
1390 alg
->cra_blocksize
= tmpl
->block_size
;
1391 p
->enc_type
= tmpl
->enc_type
;
1392 alg
->cra_ctxsize
= sizeof(struct n2_cipher_context
);
1393 alg
->cra_type
= &crypto_ablkcipher_type
;
1394 alg
->cra_u
.ablkcipher
= tmpl
->ablkcipher
;
1395 alg
->cra_init
= n2_cipher_cra_init
;
1396 alg
->cra_module
= THIS_MODULE
;
1398 list_add(&p
->entry
, &cipher_algs
);
1399 err
= crypto_register_alg(alg
);
1401 pr_err("%s alg registration failed\n", alg
->cra_name
);
1402 list_del(&p
->entry
);
1405 pr_info("%s alg registered\n", alg
->cra_name
);
1410 static int __n2_register_one_hmac(struct n2_ahash_alg
*n2ahash
)
1412 struct n2_hmac_alg
*p
= kzalloc(sizeof(*p
), GFP_KERNEL
);
1413 struct ahash_alg
*ahash
;
1414 struct crypto_alg
*base
;
1420 p
->child_alg
= n2ahash
->alg
.halg
.base
.cra_name
;
1421 memcpy(&p
->derived
, n2ahash
, sizeof(struct n2_ahash_alg
));
1422 INIT_LIST_HEAD(&p
->derived
.entry
);
1424 ahash
= &p
->derived
.alg
;
1425 ahash
->digest
= n2_hmac_async_digest
;
1426 ahash
->setkey
= n2_hmac_async_setkey
;
1428 base
= &ahash
->halg
.base
;
1429 snprintf(base
->cra_name
, CRYPTO_MAX_ALG_NAME
, "hmac(%s)", p
->child_alg
);
1430 snprintf(base
->cra_driver_name
, CRYPTO_MAX_ALG_NAME
, "hmac-%s-n2", p
->child_alg
);
1432 base
->cra_ctxsize
= sizeof(struct n2_hmac_ctx
);
1433 base
->cra_init
= n2_hmac_cra_init
;
1434 base
->cra_exit
= n2_hmac_cra_exit
;
1436 list_add(&p
->derived
.entry
, &hmac_algs
);
1437 err
= crypto_register_ahash(ahash
);
1439 pr_err("%s alg registration failed\n", base
->cra_name
);
1440 list_del(&p
->derived
.entry
);
1443 pr_info("%s alg registered\n", base
->cra_name
);
1448 static int __n2_register_one_ahash(const struct n2_hash_tmpl
*tmpl
)
1450 struct n2_ahash_alg
*p
= kzalloc(sizeof(*p
), GFP_KERNEL
);
1451 struct hash_alg_common
*halg
;
1452 struct crypto_alg
*base
;
1453 struct ahash_alg
*ahash
;
1459 p
->hash_zero
= tmpl
->hash_zero
;
1460 p
->hash_init
= tmpl
->hash_init
;
1461 p
->auth_type
= tmpl
->auth_type
;
1462 p
->hmac_type
= tmpl
->hmac_type
;
1463 p
->hw_op_hashsz
= tmpl
->hw_op_hashsz
;
1464 p
->digest_size
= tmpl
->digest_size
;
1467 ahash
->init
= n2_hash_async_init
;
1468 ahash
->update
= n2_hash_async_update
;
1469 ahash
->final
= n2_hash_async_final
;
1470 ahash
->finup
= n2_hash_async_finup
;
1471 ahash
->digest
= n2_hash_async_digest
;
1472 ahash
->export
= n2_hash_async_noexport
;
1473 ahash
->import
= n2_hash_async_noimport
;
1475 halg
= &ahash
->halg
;
1476 halg
->digestsize
= tmpl
->digest_size
;
1479 snprintf(base
->cra_name
, CRYPTO_MAX_ALG_NAME
, "%s", tmpl
->name
);
1480 snprintf(base
->cra_driver_name
, CRYPTO_MAX_ALG_NAME
, "%s-n2", tmpl
->name
);
1481 base
->cra_priority
= N2_CRA_PRIORITY
;
1482 base
->cra_flags
= CRYPTO_ALG_KERN_DRIVER_ONLY
|
1483 CRYPTO_ALG_NEED_FALLBACK
;
1484 base
->cra_blocksize
= tmpl
->block_size
;
1485 base
->cra_ctxsize
= sizeof(struct n2_hash_ctx
);
1486 base
->cra_module
= THIS_MODULE
;
1487 base
->cra_init
= n2_hash_cra_init
;
1488 base
->cra_exit
= n2_hash_cra_exit
;
1490 list_add(&p
->entry
, &ahash_algs
);
1491 err
= crypto_register_ahash(ahash
);
1493 pr_err("%s alg registration failed\n", base
->cra_name
);
1494 list_del(&p
->entry
);
1497 pr_info("%s alg registered\n", base
->cra_name
);
1499 if (!err
&& p
->hmac_type
!= AUTH_TYPE_RESERVED
)
1500 err
= __n2_register_one_hmac(p
);
1504 static int n2_register_algs(void)
1508 mutex_lock(&spu_lock
);
1509 if (algs_registered
++)
1512 for (i
= 0; i
< NUM_HASH_TMPLS
; i
++) {
1513 err
= __n2_register_one_ahash(&hash_tmpls
[i
]);
1515 __n2_unregister_algs();
1519 for (i
= 0; i
< NUM_CIPHER_TMPLS
; i
++) {
1520 err
= __n2_register_one_cipher(&cipher_tmpls
[i
]);
1522 __n2_unregister_algs();
1528 mutex_unlock(&spu_lock
);
1532 static void n2_unregister_algs(void)
1534 mutex_lock(&spu_lock
);
1535 if (!--algs_registered
)
1536 __n2_unregister_algs();
1537 mutex_unlock(&spu_lock
);
1540 /* To map CWQ queues to interrupt sources, the hypervisor API provides
1541 * a devino. This isn't very useful to us because all of the
1542 * interrupts listed in the device_node have been translated to
1543 * Linux virtual IRQ cookie numbers.
1545 * So we have to back-translate, going through the 'intr' and 'ino'
1546 * property tables of the n2cp MDESC node, matching it with the OF
1547 * 'interrupts' property entries, in order to to figure out which
1548 * devino goes to which already-translated IRQ.
1550 static int find_devino_index(struct platform_device
*dev
, struct spu_mdesc_info
*ip
,
1551 unsigned long dev_ino
)
1553 const unsigned int *dev_intrs
;
1557 for (i
= 0; i
< ip
->num_intrs
; i
++) {
1558 if (ip
->ino_table
[i
].ino
== dev_ino
)
1561 if (i
== ip
->num_intrs
)
1564 intr
= ip
->ino_table
[i
].intr
;
1566 dev_intrs
= of_get_property(dev
->dev
.of_node
, "interrupts", NULL
);
1570 for (i
= 0; i
< dev
->archdata
.num_irqs
; i
++) {
1571 if (dev_intrs
[i
] == intr
)
1578 static int spu_map_ino(struct platform_device
*dev
, struct spu_mdesc_info
*ip
,
1579 const char *irq_name
, struct spu_queue
*p
,
1580 irq_handler_t handler
)
1585 herr
= sun4v_ncs_qhandle_to_devino(p
->qhandle
, &p
->devino
);
1589 index
= find_devino_index(dev
, ip
, p
->devino
);
1593 p
->irq
= dev
->archdata
.irqs
[index
];
1595 sprintf(p
->irq_name
, "%s-%d", irq_name
, index
);
1597 return request_irq(p
->irq
, handler
, 0, p
->irq_name
, p
);
1600 static struct kmem_cache
*queue_cache
[2];
1602 static void *new_queue(unsigned long q_type
)
1604 return kmem_cache_zalloc(queue_cache
[q_type
- 1], GFP_KERNEL
);
1607 static void free_queue(void *p
, unsigned long q_type
)
1609 kmem_cache_free(queue_cache
[q_type
- 1], p
);
1612 static int queue_cache_init(void)
1614 if (!queue_cache
[HV_NCS_QTYPE_MAU
- 1])
1615 queue_cache
[HV_NCS_QTYPE_MAU
- 1] =
1616 kmem_cache_create("mau_queue",
1619 MAU_ENTRY_SIZE
, 0, NULL
);
1620 if (!queue_cache
[HV_NCS_QTYPE_MAU
- 1])
1623 if (!queue_cache
[HV_NCS_QTYPE_CWQ
- 1])
1624 queue_cache
[HV_NCS_QTYPE_CWQ
- 1] =
1625 kmem_cache_create("cwq_queue",
1628 CWQ_ENTRY_SIZE
, 0, NULL
);
1629 if (!queue_cache
[HV_NCS_QTYPE_CWQ
- 1]) {
1630 kmem_cache_destroy(queue_cache
[HV_NCS_QTYPE_MAU
- 1]);
1631 queue_cache
[HV_NCS_QTYPE_MAU
- 1] = NULL
;
1637 static void queue_cache_destroy(void)
1639 kmem_cache_destroy(queue_cache
[HV_NCS_QTYPE_MAU
- 1]);
1640 kmem_cache_destroy(queue_cache
[HV_NCS_QTYPE_CWQ
- 1]);
1641 queue_cache
[HV_NCS_QTYPE_MAU
- 1] = NULL
;
1642 queue_cache
[HV_NCS_QTYPE_CWQ
- 1] = NULL
;
1645 static long spu_queue_register_workfn(void *arg
)
1647 struct spu_qreg
*qr
= arg
;
1648 struct spu_queue
*p
= qr
->queue
;
1649 unsigned long q_type
= qr
->type
;
1650 unsigned long hv_ret
;
1652 hv_ret
= sun4v_ncs_qconf(q_type
, __pa(p
->q
),
1653 CWQ_NUM_ENTRIES
, &p
->qhandle
);
1655 sun4v_ncs_sethead_marker(p
->qhandle
, 0);
1657 return hv_ret
? -EINVAL
: 0;
1660 static int spu_queue_register(struct spu_queue
*p
, unsigned long q_type
)
1662 int cpu
= cpumask_any_and(&p
->sharing
, cpu_online_mask
);
1663 struct spu_qreg qr
= { .queue
= p
, .type
= q_type
};
1665 return work_on_cpu_safe(cpu
, spu_queue_register_workfn
, &qr
);
1668 static int spu_queue_setup(struct spu_queue
*p
)
1672 p
->q
= new_queue(p
->q_type
);
1676 err
= spu_queue_register(p
, p
->q_type
);
1678 free_queue(p
->q
, p
->q_type
);
1685 static void spu_queue_destroy(struct spu_queue
*p
)
1687 unsigned long hv_ret
;
1692 hv_ret
= sun4v_ncs_qconf(p
->q_type
, p
->qhandle
, 0, &p
->qhandle
);
1695 free_queue(p
->q
, p
->q_type
);
1698 static void spu_list_destroy(struct list_head
*list
)
1700 struct spu_queue
*p
, *n
;
1702 list_for_each_entry_safe(p
, n
, list
, list
) {
1705 for (i
= 0; i
< NR_CPUS
; i
++) {
1706 if (cpu_to_cwq
[i
] == p
)
1707 cpu_to_cwq
[i
] = NULL
;
1711 free_irq(p
->irq
, p
);
1714 spu_queue_destroy(p
);
1720 /* Walk the backward arcs of a CWQ 'exec-unit' node,
1721 * gathering cpu membership information.
1723 static int spu_mdesc_walk_arcs(struct mdesc_handle
*mdesc
,
1724 struct platform_device
*dev
,
1725 u64 node
, struct spu_queue
*p
,
1726 struct spu_queue
**table
)
1730 mdesc_for_each_arc(arc
, mdesc
, node
, MDESC_ARC_TYPE_BACK
) {
1731 u64 tgt
= mdesc_arc_target(mdesc
, arc
);
1732 const char *name
= mdesc_node_name(mdesc
, tgt
);
1735 if (strcmp(name
, "cpu"))
1737 id
= mdesc_get_property(mdesc
, tgt
, "id", NULL
);
1738 if (table
[*id
] != NULL
) {
1739 dev_err(&dev
->dev
, "%pOF: SPU cpu slot already set.\n",
1743 cpumask_set_cpu(*id
, &p
->sharing
);
1749 /* Process an 'exec-unit' MDESC node of type 'cwq'. */
1750 static int handle_exec_unit(struct spu_mdesc_info
*ip
, struct list_head
*list
,
1751 struct platform_device
*dev
, struct mdesc_handle
*mdesc
,
1752 u64 node
, const char *iname
, unsigned long q_type
,
1753 irq_handler_t handler
, struct spu_queue
**table
)
1755 struct spu_queue
*p
;
1758 p
= kzalloc(sizeof(struct spu_queue
), GFP_KERNEL
);
1760 dev_err(&dev
->dev
, "%pOF: Could not allocate SPU queue.\n",
1765 cpumask_clear(&p
->sharing
);
1766 spin_lock_init(&p
->lock
);
1768 INIT_LIST_HEAD(&p
->jobs
);
1769 list_add(&p
->list
, list
);
1771 err
= spu_mdesc_walk_arcs(mdesc
, dev
, node
, p
, table
);
1775 err
= spu_queue_setup(p
);
1779 return spu_map_ino(dev
, ip
, iname
, p
, handler
);
1782 static int spu_mdesc_scan(struct mdesc_handle
*mdesc
, struct platform_device
*dev
,
1783 struct spu_mdesc_info
*ip
, struct list_head
*list
,
1784 const char *exec_name
, unsigned long q_type
,
1785 irq_handler_t handler
, struct spu_queue
**table
)
1790 mdesc_for_each_node_by_name(mdesc
, node
, "exec-unit") {
1793 type
= mdesc_get_property(mdesc
, node
, "type", NULL
);
1794 if (!type
|| strcmp(type
, exec_name
))
1797 err
= handle_exec_unit(ip
, list
, dev
, mdesc
, node
,
1798 exec_name
, q_type
, handler
, table
);
1800 spu_list_destroy(list
);
1808 static int get_irq_props(struct mdesc_handle
*mdesc
, u64 node
,
1809 struct spu_mdesc_info
*ip
)
1815 ino
= mdesc_get_property(mdesc
, node
, "ino", &ino_len
);
1817 printk("NO 'ino'\n");
1821 ip
->num_intrs
= ino_len
/ sizeof(u64
);
1822 ip
->ino_table
= kzalloc((sizeof(struct ino_blob
) *
1828 for (i
= 0; i
< ip
->num_intrs
; i
++) {
1829 struct ino_blob
*b
= &ip
->ino_table
[i
];
1837 static int grab_mdesc_irq_props(struct mdesc_handle
*mdesc
,
1838 struct platform_device
*dev
,
1839 struct spu_mdesc_info
*ip
,
1840 const char *node_name
)
1842 const unsigned int *reg
;
1845 reg
= of_get_property(dev
->dev
.of_node
, "reg", NULL
);
1849 mdesc_for_each_node_by_name(mdesc
, node
, "virtual-device") {
1853 name
= mdesc_get_property(mdesc
, node
, "name", NULL
);
1854 if (!name
|| strcmp(name
, node_name
))
1856 chdl
= mdesc_get_property(mdesc
, node
, "cfg-handle", NULL
);
1857 if (!chdl
|| (*chdl
!= *reg
))
1859 ip
->cfg_handle
= *chdl
;
1860 return get_irq_props(mdesc
, node
, ip
);
1866 static unsigned long n2_spu_hvapi_major
;
1867 static unsigned long n2_spu_hvapi_minor
;
1869 static int n2_spu_hvapi_register(void)
1873 n2_spu_hvapi_major
= 2;
1874 n2_spu_hvapi_minor
= 0;
1876 err
= sun4v_hvapi_register(HV_GRP_NCS
,
1878 &n2_spu_hvapi_minor
);
1881 pr_info("Registered NCS HVAPI version %lu.%lu\n",
1883 n2_spu_hvapi_minor
);
1888 static void n2_spu_hvapi_unregister(void)
1890 sun4v_hvapi_unregister(HV_GRP_NCS
);
1893 static int global_ref
;
1895 static int grab_global_resources(void)
1899 mutex_lock(&spu_lock
);
1904 err
= n2_spu_hvapi_register();
1908 err
= queue_cache_init();
1910 goto out_hvapi_release
;
1913 cpu_to_cwq
= kcalloc(NR_CPUS
, sizeof(struct spu_queue
*),
1916 goto out_queue_cache_destroy
;
1918 cpu_to_mau
= kcalloc(NR_CPUS
, sizeof(struct spu_queue
*),
1921 goto out_free_cwq_table
;
1928 mutex_unlock(&spu_lock
);
1935 out_queue_cache_destroy
:
1936 queue_cache_destroy();
1939 n2_spu_hvapi_unregister();
1943 static void release_global_resources(void)
1945 mutex_lock(&spu_lock
);
1946 if (!--global_ref
) {
1953 queue_cache_destroy();
1954 n2_spu_hvapi_unregister();
1956 mutex_unlock(&spu_lock
);
1959 static struct n2_crypto
*alloc_n2cp(void)
1961 struct n2_crypto
*np
= kzalloc(sizeof(struct n2_crypto
), GFP_KERNEL
);
1964 INIT_LIST_HEAD(&np
->cwq_list
);
1969 static void free_n2cp(struct n2_crypto
*np
)
1971 kfree(np
->cwq_info
.ino_table
);
1972 np
->cwq_info
.ino_table
= NULL
;
1977 static void n2_spu_driver_version(void)
1979 static int n2_spu_version_printed
;
1981 if (n2_spu_version_printed
++ == 0)
1982 pr_info("%s", version
);
1985 static int n2_crypto_probe(struct platform_device
*dev
)
1987 struct mdesc_handle
*mdesc
;
1988 struct n2_crypto
*np
;
1991 n2_spu_driver_version();
1993 pr_info("Found N2CP at %pOF\n", dev
->dev
.of_node
);
1997 dev_err(&dev
->dev
, "%pOF: Unable to allocate n2cp.\n",
2002 err
= grab_global_resources();
2004 dev_err(&dev
->dev
, "%pOF: Unable to grab global resources.\n",
2009 mdesc
= mdesc_grab();
2012 dev_err(&dev
->dev
, "%pOF: Unable to grab MDESC.\n",
2015 goto out_free_global
;
2017 err
= grab_mdesc_irq_props(mdesc
, dev
, &np
->cwq_info
, "n2cp");
2019 dev_err(&dev
->dev
, "%pOF: Unable to grab IRQ props.\n",
2021 mdesc_release(mdesc
);
2022 goto out_free_global
;
2025 err
= spu_mdesc_scan(mdesc
, dev
, &np
->cwq_info
, &np
->cwq_list
,
2026 "cwq", HV_NCS_QTYPE_CWQ
, cwq_intr
,
2028 mdesc_release(mdesc
);
2031 dev_err(&dev
->dev
, "%pOF: CWQ MDESC scan failed.\n",
2033 goto out_free_global
;
2036 err
= n2_register_algs();
2038 dev_err(&dev
->dev
, "%pOF: Unable to register algorithms.\n",
2040 goto out_free_spu_list
;
2043 dev_set_drvdata(&dev
->dev
, np
);
2048 spu_list_destroy(&np
->cwq_list
);
2051 release_global_resources();
2059 static int n2_crypto_remove(struct platform_device
*dev
)
2061 struct n2_crypto
*np
= dev_get_drvdata(&dev
->dev
);
2063 n2_unregister_algs();
2065 spu_list_destroy(&np
->cwq_list
);
2067 release_global_resources();
2074 static struct n2_mau
*alloc_ncp(void)
2076 struct n2_mau
*mp
= kzalloc(sizeof(struct n2_mau
), GFP_KERNEL
);
2079 INIT_LIST_HEAD(&mp
->mau_list
);
2084 static void free_ncp(struct n2_mau
*mp
)
2086 kfree(mp
->mau_info
.ino_table
);
2087 mp
->mau_info
.ino_table
= NULL
;
2092 static int n2_mau_probe(struct platform_device
*dev
)
2094 struct mdesc_handle
*mdesc
;
2098 n2_spu_driver_version();
2100 pr_info("Found NCP at %pOF\n", dev
->dev
.of_node
);
2104 dev_err(&dev
->dev
, "%pOF: Unable to allocate ncp.\n",
2109 err
= grab_global_resources();
2111 dev_err(&dev
->dev
, "%pOF: Unable to grab global resources.\n",
2116 mdesc
= mdesc_grab();
2119 dev_err(&dev
->dev
, "%pOF: Unable to grab MDESC.\n",
2122 goto out_free_global
;
2125 err
= grab_mdesc_irq_props(mdesc
, dev
, &mp
->mau_info
, "ncp");
2127 dev_err(&dev
->dev
, "%pOF: Unable to grab IRQ props.\n",
2129 mdesc_release(mdesc
);
2130 goto out_free_global
;
2133 err
= spu_mdesc_scan(mdesc
, dev
, &mp
->mau_info
, &mp
->mau_list
,
2134 "mau", HV_NCS_QTYPE_MAU
, mau_intr
,
2136 mdesc_release(mdesc
);
2139 dev_err(&dev
->dev
, "%pOF: MAU MDESC scan failed.\n",
2141 goto out_free_global
;
2144 dev_set_drvdata(&dev
->dev
, mp
);
2149 release_global_resources();
2157 static int n2_mau_remove(struct platform_device
*dev
)
2159 struct n2_mau
*mp
= dev_get_drvdata(&dev
->dev
);
2161 spu_list_destroy(&mp
->mau_list
);
2163 release_global_resources();
2170 static const struct of_device_id n2_crypto_match
[] = {
2173 .compatible
= "SUNW,n2-cwq",
2177 .compatible
= "SUNW,vf-cwq",
2181 .compatible
= "SUNW,kt-cwq",
2186 MODULE_DEVICE_TABLE(of
, n2_crypto_match
);
2188 static struct platform_driver n2_crypto_driver
= {
2191 .of_match_table
= n2_crypto_match
,
2193 .probe
= n2_crypto_probe
,
2194 .remove
= n2_crypto_remove
,
2197 static const struct of_device_id n2_mau_match
[] = {
2200 .compatible
= "SUNW,n2-mau",
2204 .compatible
= "SUNW,vf-mau",
2208 .compatible
= "SUNW,kt-mau",
2213 MODULE_DEVICE_TABLE(of
, n2_mau_match
);
2215 static struct platform_driver n2_mau_driver
= {
2218 .of_match_table
= n2_mau_match
,
2220 .probe
= n2_mau_probe
,
2221 .remove
= n2_mau_remove
,
2224 static struct platform_driver
* const drivers
[] = {
2229 static int __init
n2_init(void)
2231 return platform_register_drivers(drivers
, ARRAY_SIZE(drivers
));
2234 static void __exit
n2_exit(void)
2236 platform_unregister_drivers(drivers
, ARRAY_SIZE(drivers
));
2239 module_init(n2_init
);
2240 module_exit(n2_exit
);