2 * AMD Cryptographic Coprocessor (CCP) driver
4 * Copyright (C) 2016 Advanced Micro Devices, Inc.
6 * Author: Gary R Hook <gary.hook@amd.com>
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
13 #include <linux/module.h>
14 #include <linux/kernel.h>
15 #include <linux/dmaengine.h>
16 #include <linux/spinlock.h>
17 #include <linux/mutex.h>
18 #include <linux/ccp.h>
21 #include "../../dma/dmaengine.h"
23 #define CCP_DMA_WIDTH(_mask) \
25 u64 mask = _mask + 1; \
26 (mask == 0) ? 64 : fls64(mask); \
29 /* The CCP as a DMA provider can be configured for public or private
30 * channels. Default is specified in the vdata for the device (PCI ID).
31 * This module parameter will override for all channels on all devices:
32 * dma_chan_attr = 0x2 to force all channels public
33 * = 0x1 to force all channels private
34 * = 0x0 to defer to the vdata setting
35 * = any other value: warning, revert to 0x0
37 static unsigned int dma_chan_attr
= CCP_DMA_DFLT
;
38 module_param(dma_chan_attr
, uint
, 0444);
39 MODULE_PARM_DESC(dma_chan_attr
, "Set DMA channel visibility: 0 (default) = device defaults, 1 = make private, 2 = make public");
41 unsigned int ccp_get_dma_chan_attr(struct ccp_device
*ccp
)
43 switch (dma_chan_attr
) {
45 return ccp
->vdata
->dma_chan_attr
;
54 dev_info_once(ccp
->dev
, "Invalid value for dma_chan_attr: %d\n",
56 return ccp
->vdata
->dma_chan_attr
;
60 static void ccp_free_cmd_resources(struct ccp_device
*ccp
,
61 struct list_head
*list
)
63 struct ccp_dma_cmd
*cmd
, *ctmp
;
65 list_for_each_entry_safe(cmd
, ctmp
, list
, entry
) {
66 list_del(&cmd
->entry
);
67 kmem_cache_free(ccp
->dma_cmd_cache
, cmd
);
71 static void ccp_free_desc_resources(struct ccp_device
*ccp
,
72 struct list_head
*list
)
74 struct ccp_dma_desc
*desc
, *dtmp
;
76 list_for_each_entry_safe(desc
, dtmp
, list
, entry
) {
77 ccp_free_cmd_resources(ccp
, &desc
->active
);
78 ccp_free_cmd_resources(ccp
, &desc
->pending
);
80 list_del(&desc
->entry
);
81 kmem_cache_free(ccp
->dma_desc_cache
, desc
);
85 static void ccp_free_chan_resources(struct dma_chan
*dma_chan
)
87 struct ccp_dma_chan
*chan
= container_of(dma_chan
, struct ccp_dma_chan
,
91 dev_dbg(chan
->ccp
->dev
, "%s - chan=%p\n", __func__
, chan
);
93 spin_lock_irqsave(&chan
->lock
, flags
);
95 ccp_free_desc_resources(chan
->ccp
, &chan
->complete
);
96 ccp_free_desc_resources(chan
->ccp
, &chan
->active
);
97 ccp_free_desc_resources(chan
->ccp
, &chan
->pending
);
98 ccp_free_desc_resources(chan
->ccp
, &chan
->created
);
100 spin_unlock_irqrestore(&chan
->lock
, flags
);
103 static void ccp_cleanup_desc_resources(struct ccp_device
*ccp
,
104 struct list_head
*list
)
106 struct ccp_dma_desc
*desc
, *dtmp
;
108 list_for_each_entry_safe_reverse(desc
, dtmp
, list
, entry
) {
109 if (!async_tx_test_ack(&desc
->tx_desc
))
112 dev_dbg(ccp
->dev
, "%s - desc=%p\n", __func__
, desc
);
114 ccp_free_cmd_resources(ccp
, &desc
->active
);
115 ccp_free_cmd_resources(ccp
, &desc
->pending
);
117 list_del(&desc
->entry
);
118 kmem_cache_free(ccp
->dma_desc_cache
, desc
);
122 static void ccp_do_cleanup(unsigned long data
)
124 struct ccp_dma_chan
*chan
= (struct ccp_dma_chan
*)data
;
127 dev_dbg(chan
->ccp
->dev
, "%s - chan=%s\n", __func__
,
128 dma_chan_name(&chan
->dma_chan
));
130 spin_lock_irqsave(&chan
->lock
, flags
);
132 ccp_cleanup_desc_resources(chan
->ccp
, &chan
->complete
);
134 spin_unlock_irqrestore(&chan
->lock
, flags
);
137 static int ccp_issue_next_cmd(struct ccp_dma_desc
*desc
)
139 struct ccp_dma_cmd
*cmd
;
142 cmd
= list_first_entry(&desc
->pending
, struct ccp_dma_cmd
, entry
);
143 list_move(&cmd
->entry
, &desc
->active
);
145 dev_dbg(desc
->ccp
->dev
, "%s - tx %d, cmd=%p\n", __func__
,
146 desc
->tx_desc
.cookie
, cmd
);
148 ret
= ccp_enqueue_cmd(&cmd
->ccp_cmd
);
149 if (!ret
|| (ret
== -EINPROGRESS
) || (ret
== -EBUSY
))
152 dev_dbg(desc
->ccp
->dev
, "%s - error: ret=%d, tx %d, cmd=%p\n", __func__
,
153 ret
, desc
->tx_desc
.cookie
, cmd
);
158 static void ccp_free_active_cmd(struct ccp_dma_desc
*desc
)
160 struct ccp_dma_cmd
*cmd
;
162 cmd
= list_first_entry_or_null(&desc
->active
, struct ccp_dma_cmd
,
167 dev_dbg(desc
->ccp
->dev
, "%s - freeing tx %d cmd=%p\n",
168 __func__
, desc
->tx_desc
.cookie
, cmd
);
170 list_del(&cmd
->entry
);
171 kmem_cache_free(desc
->ccp
->dma_cmd_cache
, cmd
);
174 static struct ccp_dma_desc
*__ccp_next_dma_desc(struct ccp_dma_chan
*chan
,
175 struct ccp_dma_desc
*desc
)
177 /* Move current DMA descriptor to the complete list */
179 list_move(&desc
->entry
, &chan
->complete
);
181 /* Get the next DMA descriptor on the active list */
182 desc
= list_first_entry_or_null(&chan
->active
, struct ccp_dma_desc
,
188 static struct ccp_dma_desc
*ccp_handle_active_desc(struct ccp_dma_chan
*chan
,
189 struct ccp_dma_desc
*desc
)
191 struct dma_async_tx_descriptor
*tx_desc
;
194 /* Loop over descriptors until one is found with commands */
197 /* Remove the DMA command from the list and free it */
198 ccp_free_active_cmd(desc
);
200 if (!list_empty(&desc
->pending
)) {
201 /* No errors, keep going */
202 if (desc
->status
!= DMA_ERROR
)
205 /* Error, free remaining commands and move on */
206 ccp_free_cmd_resources(desc
->ccp
,
210 tx_desc
= &desc
->tx_desc
;
215 spin_lock_irqsave(&chan
->lock
, flags
);
218 if (desc
->status
!= DMA_ERROR
)
219 desc
->status
= DMA_COMPLETE
;
221 dev_dbg(desc
->ccp
->dev
,
222 "%s - tx %d complete, status=%u\n", __func__
,
223 desc
->tx_desc
.cookie
, desc
->status
);
225 dma_cookie_complete(tx_desc
);
228 desc
= __ccp_next_dma_desc(chan
, desc
);
230 spin_unlock_irqrestore(&chan
->lock
, flags
);
233 if (tx_desc
->callback
&&
234 (tx_desc
->flags
& DMA_PREP_INTERRUPT
))
235 tx_desc
->callback(tx_desc
->callback_param
);
237 dma_run_dependencies(tx_desc
);
244 static struct ccp_dma_desc
*__ccp_pending_to_active(struct ccp_dma_chan
*chan
)
246 struct ccp_dma_desc
*desc
;
248 if (list_empty(&chan
->pending
))
251 desc
= list_empty(&chan
->active
)
252 ? list_first_entry(&chan
->pending
, struct ccp_dma_desc
, entry
)
255 list_splice_tail_init(&chan
->pending
, &chan
->active
);
260 static void ccp_cmd_callback(void *data
, int err
)
262 struct ccp_dma_desc
*desc
= data
;
263 struct ccp_dma_chan
*chan
;
266 if (err
== -EINPROGRESS
)
269 chan
= container_of(desc
->tx_desc
.chan
, struct ccp_dma_chan
,
272 dev_dbg(chan
->ccp
->dev
, "%s - tx %d callback, err=%d\n",
273 __func__
, desc
->tx_desc
.cookie
, err
);
276 desc
->status
= DMA_ERROR
;
279 /* Check for DMA descriptor completion */
280 desc
= ccp_handle_active_desc(chan
, desc
);
282 /* Don't submit cmd if no descriptor or DMA is paused */
283 if (!desc
|| (chan
->status
== DMA_PAUSED
))
286 ret
= ccp_issue_next_cmd(desc
);
290 desc
->status
= DMA_ERROR
;
293 tasklet_schedule(&chan
->cleanup_tasklet
);
296 static dma_cookie_t
ccp_tx_submit(struct dma_async_tx_descriptor
*tx_desc
)
298 struct ccp_dma_desc
*desc
= container_of(tx_desc
, struct ccp_dma_desc
,
300 struct ccp_dma_chan
*chan
;
304 chan
= container_of(tx_desc
->chan
, struct ccp_dma_chan
, dma_chan
);
306 spin_lock_irqsave(&chan
->lock
, flags
);
308 cookie
= dma_cookie_assign(tx_desc
);
309 list_del(&desc
->entry
);
310 list_add_tail(&desc
->entry
, &chan
->pending
);
312 spin_unlock_irqrestore(&chan
->lock
, flags
);
314 dev_dbg(chan
->ccp
->dev
, "%s - added tx descriptor %d to pending list\n",
320 static struct ccp_dma_cmd
*ccp_alloc_dma_cmd(struct ccp_dma_chan
*chan
)
322 struct ccp_dma_cmd
*cmd
;
324 cmd
= kmem_cache_alloc(chan
->ccp
->dma_cmd_cache
, GFP_NOWAIT
);
326 memset(cmd
, 0, sizeof(*cmd
));
331 static struct ccp_dma_desc
*ccp_alloc_dma_desc(struct ccp_dma_chan
*chan
,
334 struct ccp_dma_desc
*desc
;
336 desc
= kmem_cache_zalloc(chan
->ccp
->dma_desc_cache
, GFP_NOWAIT
);
340 dma_async_tx_descriptor_init(&desc
->tx_desc
, &chan
->dma_chan
);
341 desc
->tx_desc
.flags
= flags
;
342 desc
->tx_desc
.tx_submit
= ccp_tx_submit
;
343 desc
->ccp
= chan
->ccp
;
344 INIT_LIST_HEAD(&desc
->pending
);
345 INIT_LIST_HEAD(&desc
->active
);
346 desc
->status
= DMA_IN_PROGRESS
;
351 static struct ccp_dma_desc
*ccp_create_desc(struct dma_chan
*dma_chan
,
352 struct scatterlist
*dst_sg
,
353 unsigned int dst_nents
,
354 struct scatterlist
*src_sg
,
355 unsigned int src_nents
,
358 struct ccp_dma_chan
*chan
= container_of(dma_chan
, struct ccp_dma_chan
,
360 struct ccp_device
*ccp
= chan
->ccp
;
361 struct ccp_dma_desc
*desc
;
362 struct ccp_dma_cmd
*cmd
;
363 struct ccp_cmd
*ccp_cmd
;
364 struct ccp_passthru_nomap_engine
*ccp_pt
;
365 unsigned int src_offset
, src_len
;
366 unsigned int dst_offset
, dst_len
;
368 unsigned long sflags
;
371 if (!dst_sg
|| !src_sg
)
374 if (!dst_nents
|| !src_nents
)
377 desc
= ccp_alloc_dma_desc(chan
, flags
);
383 src_len
= sg_dma_len(src_sg
);
386 dst_len
= sg_dma_len(dst_sg
);
395 src_sg
= sg_next(src_sg
);
399 src_len
= sg_dma_len(src_sg
);
409 dst_sg
= sg_next(dst_sg
);
413 dst_len
= sg_dma_len(dst_sg
);
418 len
= min(dst_len
, src_len
);
420 cmd
= ccp_alloc_dma_cmd(chan
);
424 ccp_cmd
= &cmd
->ccp_cmd
;
425 ccp_cmd
->ccp
= chan
->ccp
;
426 ccp_pt
= &ccp_cmd
->u
.passthru_nomap
;
427 ccp_cmd
->flags
= CCP_CMD_MAY_BACKLOG
;
428 ccp_cmd
->flags
|= CCP_CMD_PASSTHRU_NO_DMA_MAP
;
429 ccp_cmd
->engine
= CCP_ENGINE_PASSTHRU
;
430 ccp_pt
->bit_mod
= CCP_PASSTHRU_BITWISE_NOOP
;
431 ccp_pt
->byte_swap
= CCP_PASSTHRU_BYTESWAP_NOOP
;
432 ccp_pt
->src_dma
= sg_dma_address(src_sg
) + src_offset
;
433 ccp_pt
->dst_dma
= sg_dma_address(dst_sg
) + dst_offset
;
434 ccp_pt
->src_len
= len
;
436 ccp_cmd
->callback
= ccp_cmd_callback
;
437 ccp_cmd
->data
= desc
;
439 list_add_tail(&cmd
->entry
, &desc
->pending
);
442 "%s - cmd=%p, src=%pad, dst=%pad, len=%llu\n", __func__
,
443 cmd
, &ccp_pt
->src_dma
,
444 &ccp_pt
->dst_dma
, ccp_pt
->src_len
);
455 desc
->len
= total_len
;
457 if (list_empty(&desc
->pending
))
460 dev_dbg(ccp
->dev
, "%s - desc=%p\n", __func__
, desc
);
462 spin_lock_irqsave(&chan
->lock
, sflags
);
464 list_add_tail(&desc
->entry
, &chan
->created
);
466 spin_unlock_irqrestore(&chan
->lock
, sflags
);
471 ccp_free_cmd_resources(ccp
, &desc
->pending
);
472 kmem_cache_free(ccp
->dma_desc_cache
, desc
);
477 static struct dma_async_tx_descriptor
*ccp_prep_dma_memcpy(
478 struct dma_chan
*dma_chan
, dma_addr_t dst
, dma_addr_t src
, size_t len
,
481 struct ccp_dma_chan
*chan
= container_of(dma_chan
, struct ccp_dma_chan
,
483 struct ccp_dma_desc
*desc
;
484 struct scatterlist dst_sg
, src_sg
;
486 dev_dbg(chan
->ccp
->dev
,
487 "%s - src=%pad, dst=%pad, len=%zu, flags=%#lx\n",
488 __func__
, &src
, &dst
, len
, flags
);
490 sg_init_table(&dst_sg
, 1);
491 sg_dma_address(&dst_sg
) = dst
;
492 sg_dma_len(&dst_sg
) = len
;
494 sg_init_table(&src_sg
, 1);
495 sg_dma_address(&src_sg
) = src
;
496 sg_dma_len(&src_sg
) = len
;
498 desc
= ccp_create_desc(dma_chan
, &dst_sg
, 1, &src_sg
, 1, flags
);
502 return &desc
->tx_desc
;
505 static struct dma_async_tx_descriptor
*ccp_prep_dma_sg(
506 struct dma_chan
*dma_chan
, struct scatterlist
*dst_sg
,
507 unsigned int dst_nents
, struct scatterlist
*src_sg
,
508 unsigned int src_nents
, unsigned long flags
)
510 struct ccp_dma_chan
*chan
= container_of(dma_chan
, struct ccp_dma_chan
,
512 struct ccp_dma_desc
*desc
;
514 dev_dbg(chan
->ccp
->dev
,
515 "%s - src=%p, src_nents=%u dst=%p, dst_nents=%u, flags=%#lx\n",
516 __func__
, src_sg
, src_nents
, dst_sg
, dst_nents
, flags
);
518 desc
= ccp_create_desc(dma_chan
, dst_sg
, dst_nents
, src_sg
, src_nents
,
523 return &desc
->tx_desc
;
526 static struct dma_async_tx_descriptor
*ccp_prep_dma_interrupt(
527 struct dma_chan
*dma_chan
, unsigned long flags
)
529 struct ccp_dma_chan
*chan
= container_of(dma_chan
, struct ccp_dma_chan
,
531 struct ccp_dma_desc
*desc
;
533 desc
= ccp_alloc_dma_desc(chan
, flags
);
537 return &desc
->tx_desc
;
540 static void ccp_issue_pending(struct dma_chan
*dma_chan
)
542 struct ccp_dma_chan
*chan
= container_of(dma_chan
, struct ccp_dma_chan
,
544 struct ccp_dma_desc
*desc
;
547 dev_dbg(chan
->ccp
->dev
, "%s\n", __func__
);
549 spin_lock_irqsave(&chan
->lock
, flags
);
551 desc
= __ccp_pending_to_active(chan
);
553 spin_unlock_irqrestore(&chan
->lock
, flags
);
555 /* If there was nothing active, start processing */
557 ccp_cmd_callback(desc
, 0);
560 static enum dma_status
ccp_tx_status(struct dma_chan
*dma_chan
,
562 struct dma_tx_state
*state
)
564 struct ccp_dma_chan
*chan
= container_of(dma_chan
, struct ccp_dma_chan
,
566 struct ccp_dma_desc
*desc
;
570 if (chan
->status
== DMA_PAUSED
) {
575 ret
= dma_cookie_status(dma_chan
, cookie
, state
);
576 if (ret
== DMA_COMPLETE
) {
577 spin_lock_irqsave(&chan
->lock
, flags
);
579 /* Get status from complete chain, if still there */
580 list_for_each_entry(desc
, &chan
->complete
, entry
) {
581 if (desc
->tx_desc
.cookie
!= cookie
)
588 spin_unlock_irqrestore(&chan
->lock
, flags
);
592 dev_dbg(chan
->ccp
->dev
, "%s - %u\n", __func__
, ret
);
597 static int ccp_pause(struct dma_chan
*dma_chan
)
599 struct ccp_dma_chan
*chan
= container_of(dma_chan
, struct ccp_dma_chan
,
602 chan
->status
= DMA_PAUSED
;
604 /*TODO: Wait for active DMA to complete before returning? */
609 static int ccp_resume(struct dma_chan
*dma_chan
)
611 struct ccp_dma_chan
*chan
= container_of(dma_chan
, struct ccp_dma_chan
,
613 struct ccp_dma_desc
*desc
;
616 spin_lock_irqsave(&chan
->lock
, flags
);
618 desc
= list_first_entry_or_null(&chan
->active
, struct ccp_dma_desc
,
621 spin_unlock_irqrestore(&chan
->lock
, flags
);
623 /* Indicate the channel is running again */
624 chan
->status
= DMA_IN_PROGRESS
;
626 /* If there was something active, re-start */
628 ccp_cmd_callback(desc
, 0);
633 static int ccp_terminate_all(struct dma_chan
*dma_chan
)
635 struct ccp_dma_chan
*chan
= container_of(dma_chan
, struct ccp_dma_chan
,
639 dev_dbg(chan
->ccp
->dev
, "%s\n", __func__
);
641 /*TODO: Wait for active DMA to complete before continuing */
643 spin_lock_irqsave(&chan
->lock
, flags
);
645 /*TODO: Purge the complete list? */
646 ccp_free_desc_resources(chan
->ccp
, &chan
->active
);
647 ccp_free_desc_resources(chan
->ccp
, &chan
->pending
);
648 ccp_free_desc_resources(chan
->ccp
, &chan
->created
);
650 spin_unlock_irqrestore(&chan
->lock
, flags
);
655 int ccp_dmaengine_register(struct ccp_device
*ccp
)
657 struct ccp_dma_chan
*chan
;
658 struct dma_device
*dma_dev
= &ccp
->dma_dev
;
659 struct dma_chan
*dma_chan
;
660 char *dma_cmd_cache_name
;
661 char *dma_desc_cache_name
;
665 ccp
->ccp_dma_chan
= devm_kcalloc(ccp
->dev
, ccp
->cmd_q_count
,
666 sizeof(*(ccp
->ccp_dma_chan
)),
668 if (!ccp
->ccp_dma_chan
)
671 dma_cmd_cache_name
= devm_kasprintf(ccp
->dev
, GFP_KERNEL
,
672 "%s-dmaengine-cmd-cache",
674 if (!dma_cmd_cache_name
)
677 ccp
->dma_cmd_cache
= kmem_cache_create(dma_cmd_cache_name
,
678 sizeof(struct ccp_dma_cmd
),
680 SLAB_HWCACHE_ALIGN
, NULL
);
681 if (!ccp
->dma_cmd_cache
)
684 dma_desc_cache_name
= devm_kasprintf(ccp
->dev
, GFP_KERNEL
,
685 "%s-dmaengine-desc-cache",
687 if (!dma_desc_cache_name
) {
692 ccp
->dma_desc_cache
= kmem_cache_create(dma_desc_cache_name
,
693 sizeof(struct ccp_dma_desc
),
695 SLAB_HWCACHE_ALIGN
, NULL
);
696 if (!ccp
->dma_desc_cache
) {
701 dma_dev
->dev
= ccp
->dev
;
702 dma_dev
->src_addr_widths
= CCP_DMA_WIDTH(dma_get_mask(ccp
->dev
));
703 dma_dev
->dst_addr_widths
= CCP_DMA_WIDTH(dma_get_mask(ccp
->dev
));
704 dma_dev
->directions
= DMA_MEM_TO_MEM
;
705 dma_dev
->residue_granularity
= DMA_RESIDUE_GRANULARITY_DESCRIPTOR
;
706 dma_cap_set(DMA_MEMCPY
, dma_dev
->cap_mask
);
707 dma_cap_set(DMA_SG
, dma_dev
->cap_mask
);
708 dma_cap_set(DMA_INTERRUPT
, dma_dev
->cap_mask
);
710 /* The DMA channels for this device can be set to public or private,
711 * and overridden by the module parameter dma_chan_attr.
712 * Default: according to the value in vdata (dma_chan_attr=0)
713 * dma_chan_attr=0x1: all channels private (override vdata)
714 * dma_chan_attr=0x2: all channels public (override vdata)
716 if (ccp_get_dma_chan_attr(ccp
) == DMA_PRIVATE
)
717 dma_cap_set(DMA_PRIVATE
, dma_dev
->cap_mask
);
719 INIT_LIST_HEAD(&dma_dev
->channels
);
720 for (i
= 0; i
< ccp
->cmd_q_count
; i
++) {
721 chan
= ccp
->ccp_dma_chan
+ i
;
722 dma_chan
= &chan
->dma_chan
;
726 spin_lock_init(&chan
->lock
);
727 INIT_LIST_HEAD(&chan
->created
);
728 INIT_LIST_HEAD(&chan
->pending
);
729 INIT_LIST_HEAD(&chan
->active
);
730 INIT_LIST_HEAD(&chan
->complete
);
732 tasklet_init(&chan
->cleanup_tasklet
, ccp_do_cleanup
,
733 (unsigned long)chan
);
735 dma_chan
->device
= dma_dev
;
736 dma_cookie_init(dma_chan
);
738 list_add_tail(&dma_chan
->device_node
, &dma_dev
->channels
);
741 dma_dev
->device_free_chan_resources
= ccp_free_chan_resources
;
742 dma_dev
->device_prep_dma_memcpy
= ccp_prep_dma_memcpy
;
743 dma_dev
->device_prep_dma_sg
= ccp_prep_dma_sg
;
744 dma_dev
->device_prep_dma_interrupt
= ccp_prep_dma_interrupt
;
745 dma_dev
->device_issue_pending
= ccp_issue_pending
;
746 dma_dev
->device_tx_status
= ccp_tx_status
;
747 dma_dev
->device_pause
= ccp_pause
;
748 dma_dev
->device_resume
= ccp_resume
;
749 dma_dev
->device_terminate_all
= ccp_terminate_all
;
751 ret
= dma_async_device_register(dma_dev
);
758 kmem_cache_destroy(ccp
->dma_desc_cache
);
761 kmem_cache_destroy(ccp
->dma_cmd_cache
);
766 void ccp_dmaengine_unregister(struct ccp_device
*ccp
)
768 struct dma_device
*dma_dev
= &ccp
->dma_dev
;
770 dma_async_device_unregister(dma_dev
);
772 kmem_cache_destroy(ccp
->dma_desc_cache
);
773 kmem_cache_destroy(ccp
->dma_cmd_cache
);