]>
Commit | Line | Data |
---|---|---|
1 | // SPDX-License-Identifier: GPL-2.0-only | |
2 | /* | |
3 | * AMD Cryptographic Coprocessor (CCP) driver | |
4 | * | |
5 | * Copyright (C) 2016,2019 Advanced Micro Devices, Inc. | |
6 | * | |
7 | * Author: Gary R Hook <gary.hook@amd.com> | |
8 | */ | |
9 | ||
10 | #include <linux/module.h> | |
11 | #include <linux/kernel.h> | |
12 | #include <linux/dma-mapping.h> | |
13 | #include <linux/dmaengine.h> | |
14 | #include <linux/spinlock.h> | |
15 | #include <linux/mutex.h> | |
16 | #include <linux/ccp.h> | |
17 | ||
18 | #include "ccp-dev.h" | |
19 | #include "../../dma/dmaengine.h" | |
20 | ||
21 | #define CCP_DMA_WIDTH(_mask) \ | |
22 | ({ \ | |
23 | u64 mask = _mask + 1; \ | |
24 | (mask == 0) ? 64 : fls64(mask); \ | |
25 | }) | |
26 | ||
27 | /* The CCP as a DMA provider can be configured for public or private | |
28 | * channels. Default is specified in the vdata for the device (PCI ID). | |
29 | * This module parameter will override for all channels on all devices: | |
30 | * dma_chan_attr = 0x2 to force all channels public | |
31 | * = 0x1 to force all channels private | |
32 | * = 0x0 to defer to the vdata setting | |
33 | * = any other value: warning, revert to 0x0 | |
34 | */ | |
35 | static unsigned int dma_chan_attr = CCP_DMA_DFLT; | |
36 | module_param(dma_chan_attr, uint, 0444); | |
37 | MODULE_PARM_DESC(dma_chan_attr, "Set DMA channel visibility: 0 (default) = device defaults, 1 = make private, 2 = make public"); | |
38 | ||
39 | static unsigned int dmaengine = 1; | |
40 | module_param(dmaengine, uint, 0444); | |
41 | MODULE_PARM_DESC(dmaengine, "Register services with the DMA subsystem (any non-zero value, default: 1)"); | |
42 | ||
43 | static unsigned int ccp_get_dma_chan_attr(struct ccp_device *ccp) | |
44 | { | |
45 | switch (dma_chan_attr) { | |
46 | case CCP_DMA_DFLT: | |
47 | return ccp->vdata->dma_chan_attr; | |
48 | ||
49 | case CCP_DMA_PRIV: | |
50 | return DMA_PRIVATE; | |
51 | ||
52 | case CCP_DMA_PUB: | |
53 | return 0; | |
54 | ||
55 | default: | |
56 | dev_info_once(ccp->dev, "Invalid value for dma_chan_attr: %d\n", | |
57 | dma_chan_attr); | |
58 | return ccp->vdata->dma_chan_attr; | |
59 | } | |
60 | } | |
61 | ||
62 | static void ccp_free_cmd_resources(struct ccp_device *ccp, | |
63 | struct list_head *list) | |
64 | { | |
65 | struct ccp_dma_cmd *cmd, *ctmp; | |
66 | ||
67 | list_for_each_entry_safe(cmd, ctmp, list, entry) { | |
68 | list_del(&cmd->entry); | |
69 | kmem_cache_free(ccp->dma_cmd_cache, cmd); | |
70 | } | |
71 | } | |
72 | ||
73 | static void ccp_free_desc_resources(struct ccp_device *ccp, | |
74 | struct list_head *list) | |
75 | { | |
76 | struct ccp_dma_desc *desc, *dtmp; | |
77 | ||
78 | list_for_each_entry_safe(desc, dtmp, list, entry) { | |
79 | ccp_free_cmd_resources(ccp, &desc->active); | |
80 | ccp_free_cmd_resources(ccp, &desc->pending); | |
81 | ||
82 | list_del(&desc->entry); | |
83 | kmem_cache_free(ccp->dma_desc_cache, desc); | |
84 | } | |
85 | } | |
86 | ||
87 | static void ccp_free_chan_resources(struct dma_chan *dma_chan) | |
88 | { | |
89 | struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, | |
90 | dma_chan); | |
91 | unsigned long flags; | |
92 | ||
93 | dev_dbg(chan->ccp->dev, "%s - chan=%p\n", __func__, chan); | |
94 | ||
95 | spin_lock_irqsave(&chan->lock, flags); | |
96 | ||
97 | ccp_free_desc_resources(chan->ccp, &chan->complete); | |
98 | ccp_free_desc_resources(chan->ccp, &chan->active); | |
99 | ccp_free_desc_resources(chan->ccp, &chan->pending); | |
100 | ccp_free_desc_resources(chan->ccp, &chan->created); | |
101 | ||
102 | spin_unlock_irqrestore(&chan->lock, flags); | |
103 | } | |
104 | ||
105 | static void ccp_cleanup_desc_resources(struct ccp_device *ccp, | |
106 | struct list_head *list) | |
107 | { | |
108 | struct ccp_dma_desc *desc, *dtmp; | |
109 | ||
110 | list_for_each_entry_safe_reverse(desc, dtmp, list, entry) { | |
111 | if (!async_tx_test_ack(&desc->tx_desc)) | |
112 | continue; | |
113 | ||
114 | dev_dbg(ccp->dev, "%s - desc=%p\n", __func__, desc); | |
115 | ||
116 | ccp_free_cmd_resources(ccp, &desc->active); | |
117 | ccp_free_cmd_resources(ccp, &desc->pending); | |
118 | ||
119 | list_del(&desc->entry); | |
120 | kmem_cache_free(ccp->dma_desc_cache, desc); | |
121 | } | |
122 | } | |
123 | ||
124 | static void ccp_do_cleanup(unsigned long data) | |
125 | { | |
126 | struct ccp_dma_chan *chan = (struct ccp_dma_chan *)data; | |
127 | unsigned long flags; | |
128 | ||
129 | dev_dbg(chan->ccp->dev, "%s - chan=%s\n", __func__, | |
130 | dma_chan_name(&chan->dma_chan)); | |
131 | ||
132 | spin_lock_irqsave(&chan->lock, flags); | |
133 | ||
134 | ccp_cleanup_desc_resources(chan->ccp, &chan->complete); | |
135 | ||
136 | spin_unlock_irqrestore(&chan->lock, flags); | |
137 | } | |
138 | ||
139 | static int ccp_issue_next_cmd(struct ccp_dma_desc *desc) | |
140 | { | |
141 | struct ccp_dma_cmd *cmd; | |
142 | int ret; | |
143 | ||
144 | cmd = list_first_entry(&desc->pending, struct ccp_dma_cmd, entry); | |
145 | list_move(&cmd->entry, &desc->active); | |
146 | ||
147 | dev_dbg(desc->ccp->dev, "%s - tx %d, cmd=%p\n", __func__, | |
148 | desc->tx_desc.cookie, cmd); | |
149 | ||
150 | ret = ccp_enqueue_cmd(&cmd->ccp_cmd); | |
151 | if (!ret || (ret == -EINPROGRESS) || (ret == -EBUSY)) | |
152 | return 0; | |
153 | ||
154 | dev_dbg(desc->ccp->dev, "%s - error: ret=%d, tx %d, cmd=%p\n", __func__, | |
155 | ret, desc->tx_desc.cookie, cmd); | |
156 | ||
157 | return ret; | |
158 | } | |
159 | ||
160 | static void ccp_free_active_cmd(struct ccp_dma_desc *desc) | |
161 | { | |
162 | struct ccp_dma_cmd *cmd; | |
163 | ||
164 | cmd = list_first_entry_or_null(&desc->active, struct ccp_dma_cmd, | |
165 | entry); | |
166 | if (!cmd) | |
167 | return; | |
168 | ||
169 | dev_dbg(desc->ccp->dev, "%s - freeing tx %d cmd=%p\n", | |
170 | __func__, desc->tx_desc.cookie, cmd); | |
171 | ||
172 | list_del(&cmd->entry); | |
173 | kmem_cache_free(desc->ccp->dma_cmd_cache, cmd); | |
174 | } | |
175 | ||
176 | static struct ccp_dma_desc *__ccp_next_dma_desc(struct ccp_dma_chan *chan, | |
177 | struct ccp_dma_desc *desc) | |
178 | { | |
179 | /* Move current DMA descriptor to the complete list */ | |
180 | if (desc) | |
181 | list_move(&desc->entry, &chan->complete); | |
182 | ||
183 | /* Get the next DMA descriptor on the active list */ | |
184 | desc = list_first_entry_or_null(&chan->active, struct ccp_dma_desc, | |
185 | entry); | |
186 | ||
187 | return desc; | |
188 | } | |
189 | ||
190 | static struct ccp_dma_desc *ccp_handle_active_desc(struct ccp_dma_chan *chan, | |
191 | struct ccp_dma_desc *desc) | |
192 | { | |
193 | struct dma_async_tx_descriptor *tx_desc; | |
194 | unsigned long flags; | |
195 | ||
196 | /* Loop over descriptors until one is found with commands */ | |
197 | do { | |
198 | if (desc) { | |
199 | /* Remove the DMA command from the list and free it */ | |
200 | ccp_free_active_cmd(desc); | |
201 | ||
202 | if (!list_empty(&desc->pending)) { | |
203 | /* No errors, keep going */ | |
204 | if (desc->status != DMA_ERROR) | |
205 | return desc; | |
206 | ||
207 | /* Error, free remaining commands and move on */ | |
208 | ccp_free_cmd_resources(desc->ccp, | |
209 | &desc->pending); | |
210 | } | |
211 | ||
212 | tx_desc = &desc->tx_desc; | |
213 | } else { | |
214 | tx_desc = NULL; | |
215 | } | |
216 | ||
217 | spin_lock_irqsave(&chan->lock, flags); | |
218 | ||
219 | if (desc) { | |
220 | if (desc->status != DMA_ERROR) | |
221 | desc->status = DMA_COMPLETE; | |
222 | ||
223 | dev_dbg(desc->ccp->dev, | |
224 | "%s - tx %d complete, status=%u\n", __func__, | |
225 | desc->tx_desc.cookie, desc->status); | |
226 | ||
227 | dma_cookie_complete(tx_desc); | |
228 | dma_descriptor_unmap(tx_desc); | |
229 | } | |
230 | ||
231 | desc = __ccp_next_dma_desc(chan, desc); | |
232 | ||
233 | spin_unlock_irqrestore(&chan->lock, flags); | |
234 | ||
235 | if (tx_desc) { | |
236 | dmaengine_desc_get_callback_invoke(tx_desc, NULL); | |
237 | ||
238 | dma_run_dependencies(tx_desc); | |
239 | } | |
240 | } while (desc); | |
241 | ||
242 | return NULL; | |
243 | } | |
244 | ||
245 | static struct ccp_dma_desc *__ccp_pending_to_active(struct ccp_dma_chan *chan) | |
246 | { | |
247 | struct ccp_dma_desc *desc; | |
248 | ||
249 | if (list_empty(&chan->pending)) | |
250 | return NULL; | |
251 | ||
252 | desc = list_empty(&chan->active) | |
253 | ? list_first_entry(&chan->pending, struct ccp_dma_desc, entry) | |
254 | : NULL; | |
255 | ||
256 | list_splice_tail_init(&chan->pending, &chan->active); | |
257 | ||
258 | return desc; | |
259 | } | |
260 | ||
261 | static void ccp_cmd_callback(void *data, int err) | |
262 | { | |
263 | struct ccp_dma_desc *desc = data; | |
264 | struct ccp_dma_chan *chan; | |
265 | int ret; | |
266 | ||
267 | if (err == -EINPROGRESS) | |
268 | return; | |
269 | ||
270 | chan = container_of(desc->tx_desc.chan, struct ccp_dma_chan, | |
271 | dma_chan); | |
272 | ||
273 | dev_dbg(chan->ccp->dev, "%s - tx %d callback, err=%d\n", | |
274 | __func__, desc->tx_desc.cookie, err); | |
275 | ||
276 | if (err) | |
277 | desc->status = DMA_ERROR; | |
278 | ||
279 | while (true) { | |
280 | /* Check for DMA descriptor completion */ | |
281 | desc = ccp_handle_active_desc(chan, desc); | |
282 | ||
283 | /* Don't submit cmd if no descriptor or DMA is paused */ | |
284 | if (!desc || (chan->status == DMA_PAUSED)) | |
285 | break; | |
286 | ||
287 | ret = ccp_issue_next_cmd(desc); | |
288 | if (!ret) | |
289 | break; | |
290 | ||
291 | desc->status = DMA_ERROR; | |
292 | } | |
293 | ||
294 | tasklet_schedule(&chan->cleanup_tasklet); | |
295 | } | |
296 | ||
297 | static dma_cookie_t ccp_tx_submit(struct dma_async_tx_descriptor *tx_desc) | |
298 | { | |
299 | struct ccp_dma_desc *desc = container_of(tx_desc, struct ccp_dma_desc, | |
300 | tx_desc); | |
301 | struct ccp_dma_chan *chan; | |
302 | dma_cookie_t cookie; | |
303 | unsigned long flags; | |
304 | ||
305 | chan = container_of(tx_desc->chan, struct ccp_dma_chan, dma_chan); | |
306 | ||
307 | spin_lock_irqsave(&chan->lock, flags); | |
308 | ||
309 | cookie = dma_cookie_assign(tx_desc); | |
310 | list_del(&desc->entry); | |
311 | list_add_tail(&desc->entry, &chan->pending); | |
312 | ||
313 | spin_unlock_irqrestore(&chan->lock, flags); | |
314 | ||
315 | dev_dbg(chan->ccp->dev, "%s - added tx descriptor %d to pending list\n", | |
316 | __func__, cookie); | |
317 | ||
318 | return cookie; | |
319 | } | |
320 | ||
321 | static struct ccp_dma_cmd *ccp_alloc_dma_cmd(struct ccp_dma_chan *chan) | |
322 | { | |
323 | struct ccp_dma_cmd *cmd; | |
324 | ||
325 | cmd = kmem_cache_alloc(chan->ccp->dma_cmd_cache, GFP_NOWAIT); | |
326 | if (cmd) | |
327 | memset(cmd, 0, sizeof(*cmd)); | |
328 | ||
329 | return cmd; | |
330 | } | |
331 | ||
332 | static struct ccp_dma_desc *ccp_alloc_dma_desc(struct ccp_dma_chan *chan, | |
333 | unsigned long flags) | |
334 | { | |
335 | struct ccp_dma_desc *desc; | |
336 | ||
337 | desc = kmem_cache_zalloc(chan->ccp->dma_desc_cache, GFP_NOWAIT); | |
338 | if (!desc) | |
339 | return NULL; | |
340 | ||
341 | dma_async_tx_descriptor_init(&desc->tx_desc, &chan->dma_chan); | |
342 | desc->tx_desc.flags = flags; | |
343 | desc->tx_desc.tx_submit = ccp_tx_submit; | |
344 | desc->ccp = chan->ccp; | |
345 | INIT_LIST_HEAD(&desc->entry); | |
346 | INIT_LIST_HEAD(&desc->pending); | |
347 | INIT_LIST_HEAD(&desc->active); | |
348 | desc->status = DMA_IN_PROGRESS; | |
349 | ||
350 | return desc; | |
351 | } | |
352 | ||
353 | static struct ccp_dma_desc *ccp_create_desc(struct dma_chan *dma_chan, | |
354 | struct scatterlist *dst_sg, | |
355 | unsigned int dst_nents, | |
356 | struct scatterlist *src_sg, | |
357 | unsigned int src_nents, | |
358 | unsigned long flags) | |
359 | { | |
360 | struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, | |
361 | dma_chan); | |
362 | struct ccp_device *ccp = chan->ccp; | |
363 | struct ccp_dma_desc *desc; | |
364 | struct ccp_dma_cmd *cmd; | |
365 | struct ccp_cmd *ccp_cmd; | |
366 | struct ccp_passthru_nomap_engine *ccp_pt; | |
367 | unsigned int src_offset, src_len; | |
368 | unsigned int dst_offset, dst_len; | |
369 | unsigned int len; | |
370 | unsigned long sflags; | |
371 | size_t total_len; | |
372 | ||
373 | if (!dst_sg || !src_sg) | |
374 | return NULL; | |
375 | ||
376 | if (!dst_nents || !src_nents) | |
377 | return NULL; | |
378 | ||
379 | desc = ccp_alloc_dma_desc(chan, flags); | |
380 | if (!desc) | |
381 | return NULL; | |
382 | ||
383 | total_len = 0; | |
384 | ||
385 | src_len = sg_dma_len(src_sg); | |
386 | src_offset = 0; | |
387 | ||
388 | dst_len = sg_dma_len(dst_sg); | |
389 | dst_offset = 0; | |
390 | ||
391 | while (true) { | |
392 | if (!src_len) { | |
393 | src_nents--; | |
394 | if (!src_nents) | |
395 | break; | |
396 | ||
397 | src_sg = sg_next(src_sg); | |
398 | if (!src_sg) | |
399 | break; | |
400 | ||
401 | src_len = sg_dma_len(src_sg); | |
402 | src_offset = 0; | |
403 | continue; | |
404 | } | |
405 | ||
406 | if (!dst_len) { | |
407 | dst_nents--; | |
408 | if (!dst_nents) | |
409 | break; | |
410 | ||
411 | dst_sg = sg_next(dst_sg); | |
412 | if (!dst_sg) | |
413 | break; | |
414 | ||
415 | dst_len = sg_dma_len(dst_sg); | |
416 | dst_offset = 0; | |
417 | continue; | |
418 | } | |
419 | ||
420 | len = min(dst_len, src_len); | |
421 | ||
422 | cmd = ccp_alloc_dma_cmd(chan); | |
423 | if (!cmd) | |
424 | goto err; | |
425 | ||
426 | ccp_cmd = &cmd->ccp_cmd; | |
427 | ccp_cmd->ccp = chan->ccp; | |
428 | ccp_pt = &ccp_cmd->u.passthru_nomap; | |
429 | ccp_cmd->flags = CCP_CMD_MAY_BACKLOG; | |
430 | ccp_cmd->flags |= CCP_CMD_PASSTHRU_NO_DMA_MAP; | |
431 | ccp_cmd->engine = CCP_ENGINE_PASSTHRU; | |
432 | ccp_pt->bit_mod = CCP_PASSTHRU_BITWISE_NOOP; | |
433 | ccp_pt->byte_swap = CCP_PASSTHRU_BYTESWAP_NOOP; | |
434 | ccp_pt->src_dma = sg_dma_address(src_sg) + src_offset; | |
435 | ccp_pt->dst_dma = sg_dma_address(dst_sg) + dst_offset; | |
436 | ccp_pt->src_len = len; | |
437 | ccp_pt->final = 1; | |
438 | ccp_cmd->callback = ccp_cmd_callback; | |
439 | ccp_cmd->data = desc; | |
440 | ||
441 | list_add_tail(&cmd->entry, &desc->pending); | |
442 | ||
443 | dev_dbg(ccp->dev, | |
444 | "%s - cmd=%p, src=%pad, dst=%pad, len=%llu\n", __func__, | |
445 | cmd, &ccp_pt->src_dma, | |
446 | &ccp_pt->dst_dma, ccp_pt->src_len); | |
447 | ||
448 | total_len += len; | |
449 | ||
450 | src_len -= len; | |
451 | src_offset += len; | |
452 | ||
453 | dst_len -= len; | |
454 | dst_offset += len; | |
455 | } | |
456 | ||
457 | desc->len = total_len; | |
458 | ||
459 | if (list_empty(&desc->pending)) | |
460 | goto err; | |
461 | ||
462 | dev_dbg(ccp->dev, "%s - desc=%p\n", __func__, desc); | |
463 | ||
464 | spin_lock_irqsave(&chan->lock, sflags); | |
465 | ||
466 | list_add_tail(&desc->entry, &chan->created); | |
467 | ||
468 | spin_unlock_irqrestore(&chan->lock, sflags); | |
469 | ||
470 | return desc; | |
471 | ||
472 | err: | |
473 | ccp_free_cmd_resources(ccp, &desc->pending); | |
474 | kmem_cache_free(ccp->dma_desc_cache, desc); | |
475 | ||
476 | return NULL; | |
477 | } | |
478 | ||
479 | static struct dma_async_tx_descriptor *ccp_prep_dma_memcpy( | |
480 | struct dma_chan *dma_chan, dma_addr_t dst, dma_addr_t src, size_t len, | |
481 | unsigned long flags) | |
482 | { | |
483 | struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, | |
484 | dma_chan); | |
485 | struct ccp_dma_desc *desc; | |
486 | struct scatterlist dst_sg, src_sg; | |
487 | ||
488 | dev_dbg(chan->ccp->dev, | |
489 | "%s - src=%pad, dst=%pad, len=%zu, flags=%#lx\n", | |
490 | __func__, &src, &dst, len, flags); | |
491 | ||
492 | sg_init_table(&dst_sg, 1); | |
493 | sg_dma_address(&dst_sg) = dst; | |
494 | sg_dma_len(&dst_sg) = len; | |
495 | ||
496 | sg_init_table(&src_sg, 1); | |
497 | sg_dma_address(&src_sg) = src; | |
498 | sg_dma_len(&src_sg) = len; | |
499 | ||
500 | desc = ccp_create_desc(dma_chan, &dst_sg, 1, &src_sg, 1, flags); | |
501 | if (!desc) | |
502 | return NULL; | |
503 | ||
504 | return &desc->tx_desc; | |
505 | } | |
506 | ||
507 | static struct dma_async_tx_descriptor *ccp_prep_dma_interrupt( | |
508 | struct dma_chan *dma_chan, unsigned long flags) | |
509 | { | |
510 | struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, | |
511 | dma_chan); | |
512 | struct ccp_dma_desc *desc; | |
513 | ||
514 | desc = ccp_alloc_dma_desc(chan, flags); | |
515 | if (!desc) | |
516 | return NULL; | |
517 | ||
518 | return &desc->tx_desc; | |
519 | } | |
520 | ||
521 | static void ccp_issue_pending(struct dma_chan *dma_chan) | |
522 | { | |
523 | struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, | |
524 | dma_chan); | |
525 | struct ccp_dma_desc *desc; | |
526 | unsigned long flags; | |
527 | ||
528 | dev_dbg(chan->ccp->dev, "%s\n", __func__); | |
529 | ||
530 | spin_lock_irqsave(&chan->lock, flags); | |
531 | ||
532 | desc = __ccp_pending_to_active(chan); | |
533 | ||
534 | spin_unlock_irqrestore(&chan->lock, flags); | |
535 | ||
536 | /* If there was nothing active, start processing */ | |
537 | if (desc) | |
538 | ccp_cmd_callback(desc, 0); | |
539 | } | |
540 | ||
541 | static enum dma_status ccp_tx_status(struct dma_chan *dma_chan, | |
542 | dma_cookie_t cookie, | |
543 | struct dma_tx_state *state) | |
544 | { | |
545 | struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, | |
546 | dma_chan); | |
547 | struct ccp_dma_desc *desc; | |
548 | enum dma_status ret; | |
549 | unsigned long flags; | |
550 | ||
551 | if (chan->status == DMA_PAUSED) { | |
552 | ret = DMA_PAUSED; | |
553 | goto out; | |
554 | } | |
555 | ||
556 | ret = dma_cookie_status(dma_chan, cookie, state); | |
557 | if (ret == DMA_COMPLETE) { | |
558 | spin_lock_irqsave(&chan->lock, flags); | |
559 | ||
560 | /* Get status from complete chain, if still there */ | |
561 | list_for_each_entry(desc, &chan->complete, entry) { | |
562 | if (desc->tx_desc.cookie != cookie) | |
563 | continue; | |
564 | ||
565 | ret = desc->status; | |
566 | break; | |
567 | } | |
568 | ||
569 | spin_unlock_irqrestore(&chan->lock, flags); | |
570 | } | |
571 | ||
572 | out: | |
573 | dev_dbg(chan->ccp->dev, "%s - %u\n", __func__, ret); | |
574 | ||
575 | return ret; | |
576 | } | |
577 | ||
578 | static int ccp_pause(struct dma_chan *dma_chan) | |
579 | { | |
580 | struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, | |
581 | dma_chan); | |
582 | ||
583 | chan->status = DMA_PAUSED; | |
584 | ||
585 | /*TODO: Wait for active DMA to complete before returning? */ | |
586 | ||
587 | return 0; | |
588 | } | |
589 | ||
590 | static int ccp_resume(struct dma_chan *dma_chan) | |
591 | { | |
592 | struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, | |
593 | dma_chan); | |
594 | struct ccp_dma_desc *desc; | |
595 | unsigned long flags; | |
596 | ||
597 | spin_lock_irqsave(&chan->lock, flags); | |
598 | ||
599 | desc = list_first_entry_or_null(&chan->active, struct ccp_dma_desc, | |
600 | entry); | |
601 | ||
602 | spin_unlock_irqrestore(&chan->lock, flags); | |
603 | ||
604 | /* Indicate the channel is running again */ | |
605 | chan->status = DMA_IN_PROGRESS; | |
606 | ||
607 | /* If there was something active, re-start */ | |
608 | if (desc) | |
609 | ccp_cmd_callback(desc, 0); | |
610 | ||
611 | return 0; | |
612 | } | |
613 | ||
614 | static int ccp_terminate_all(struct dma_chan *dma_chan) | |
615 | { | |
616 | struct ccp_dma_chan *chan = container_of(dma_chan, struct ccp_dma_chan, | |
617 | dma_chan); | |
618 | unsigned long flags; | |
619 | ||
620 | dev_dbg(chan->ccp->dev, "%s\n", __func__); | |
621 | ||
622 | /*TODO: Wait for active DMA to complete before continuing */ | |
623 | ||
624 | spin_lock_irqsave(&chan->lock, flags); | |
625 | ||
626 | /*TODO: Purge the complete list? */ | |
627 | ccp_free_desc_resources(chan->ccp, &chan->active); | |
628 | ccp_free_desc_resources(chan->ccp, &chan->pending); | |
629 | ccp_free_desc_resources(chan->ccp, &chan->created); | |
630 | ||
631 | spin_unlock_irqrestore(&chan->lock, flags); | |
632 | ||
633 | return 0; | |
634 | } | |
635 | ||
636 | int ccp_dmaengine_register(struct ccp_device *ccp) | |
637 | { | |
638 | struct ccp_dma_chan *chan; | |
639 | struct dma_device *dma_dev = &ccp->dma_dev; | |
640 | struct dma_chan *dma_chan; | |
641 | char *dma_cmd_cache_name; | |
642 | char *dma_desc_cache_name; | |
643 | unsigned int i; | |
644 | int ret; | |
645 | ||
646 | if (!dmaengine) | |
647 | return 0; | |
648 | ||
649 | ccp->ccp_dma_chan = devm_kcalloc(ccp->dev, ccp->cmd_q_count, | |
650 | sizeof(*(ccp->ccp_dma_chan)), | |
651 | GFP_KERNEL); | |
652 | if (!ccp->ccp_dma_chan) | |
653 | return -ENOMEM; | |
654 | ||
655 | dma_cmd_cache_name = devm_kasprintf(ccp->dev, GFP_KERNEL, | |
656 | "%s-dmaengine-cmd-cache", | |
657 | ccp->name); | |
658 | if (!dma_cmd_cache_name) | |
659 | return -ENOMEM; | |
660 | ||
661 | ccp->dma_cmd_cache = kmem_cache_create(dma_cmd_cache_name, | |
662 | sizeof(struct ccp_dma_cmd), | |
663 | sizeof(void *), | |
664 | SLAB_HWCACHE_ALIGN, NULL); | |
665 | if (!ccp->dma_cmd_cache) | |
666 | return -ENOMEM; | |
667 | ||
668 | dma_desc_cache_name = devm_kasprintf(ccp->dev, GFP_KERNEL, | |
669 | "%s-dmaengine-desc-cache", | |
670 | ccp->name); | |
671 | if (!dma_desc_cache_name) { | |
672 | ret = -ENOMEM; | |
673 | goto err_cache; | |
674 | } | |
675 | ||
676 | ccp->dma_desc_cache = kmem_cache_create(dma_desc_cache_name, | |
677 | sizeof(struct ccp_dma_desc), | |
678 | sizeof(void *), | |
679 | SLAB_HWCACHE_ALIGN, NULL); | |
680 | if (!ccp->dma_desc_cache) { | |
681 | ret = -ENOMEM; | |
682 | goto err_cache; | |
683 | } | |
684 | ||
685 | dma_dev->dev = ccp->dev; | |
686 | dma_dev->src_addr_widths = CCP_DMA_WIDTH(dma_get_mask(ccp->dev)); | |
687 | dma_dev->dst_addr_widths = CCP_DMA_WIDTH(dma_get_mask(ccp->dev)); | |
688 | dma_dev->directions = DMA_MEM_TO_MEM; | |
689 | dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; | |
690 | dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask); | |
691 | dma_cap_set(DMA_INTERRUPT, dma_dev->cap_mask); | |
692 | ||
693 | /* The DMA channels for this device can be set to public or private, | |
694 | * and overridden by the module parameter dma_chan_attr. | |
695 | * Default: according to the value in vdata (dma_chan_attr=0) | |
696 | * dma_chan_attr=0x1: all channels private (override vdata) | |
697 | * dma_chan_attr=0x2: all channels public (override vdata) | |
698 | */ | |
699 | if (ccp_get_dma_chan_attr(ccp) == DMA_PRIVATE) | |
700 | dma_cap_set(DMA_PRIVATE, dma_dev->cap_mask); | |
701 | ||
702 | INIT_LIST_HEAD(&dma_dev->channels); | |
703 | for (i = 0; i < ccp->cmd_q_count; i++) { | |
704 | chan = ccp->ccp_dma_chan + i; | |
705 | dma_chan = &chan->dma_chan; | |
706 | ||
707 | chan->ccp = ccp; | |
708 | ||
709 | spin_lock_init(&chan->lock); | |
710 | INIT_LIST_HEAD(&chan->created); | |
711 | INIT_LIST_HEAD(&chan->pending); | |
712 | INIT_LIST_HEAD(&chan->active); | |
713 | INIT_LIST_HEAD(&chan->complete); | |
714 | ||
715 | tasklet_init(&chan->cleanup_tasklet, ccp_do_cleanup, | |
716 | (unsigned long)chan); | |
717 | ||
718 | dma_chan->device = dma_dev; | |
719 | dma_cookie_init(dma_chan); | |
720 | ||
721 | list_add_tail(&dma_chan->device_node, &dma_dev->channels); | |
722 | } | |
723 | ||
724 | dma_dev->device_free_chan_resources = ccp_free_chan_resources; | |
725 | dma_dev->device_prep_dma_memcpy = ccp_prep_dma_memcpy; | |
726 | dma_dev->device_prep_dma_interrupt = ccp_prep_dma_interrupt; | |
727 | dma_dev->device_issue_pending = ccp_issue_pending; | |
728 | dma_dev->device_tx_status = ccp_tx_status; | |
729 | dma_dev->device_pause = ccp_pause; | |
730 | dma_dev->device_resume = ccp_resume; | |
731 | dma_dev->device_terminate_all = ccp_terminate_all; | |
732 | ||
733 | ret = dma_async_device_register(dma_dev); | |
734 | if (ret) | |
735 | goto err_reg; | |
736 | ||
737 | return 0; | |
738 | ||
739 | err_reg: | |
740 | kmem_cache_destroy(ccp->dma_desc_cache); | |
741 | ||
742 | err_cache: | |
743 | kmem_cache_destroy(ccp->dma_cmd_cache); | |
744 | ||
745 | return ret; | |
746 | } | |
747 | ||
748 | void ccp_dmaengine_unregister(struct ccp_device *ccp) | |
749 | { | |
750 | struct dma_device *dma_dev = &ccp->dma_dev; | |
751 | ||
752 | if (!dmaengine) | |
753 | return; | |
754 | ||
755 | dma_async_device_unregister(dma_dev); | |
756 | ||
757 | kmem_cache_destroy(ccp->dma_desc_cache); | |
758 | kmem_cache_destroy(ccp->dma_cmd_cache); | |
759 | } |