]>
Commit | Line | Data |
---|---|---|
c13c8260 CL |
1 | /* |
2 | * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved. | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or modify it | |
5 | * under the terms of the GNU General Public License as published by the Free | |
6 | * Software Foundation; either version 2 of the License, or (at your option) | |
7 | * any later version. | |
8 | * | |
9 | * This program is distributed in the hope that it will be useful, but WITHOUT | |
10 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
11 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | |
12 | * more details. | |
13 | * | |
14 | * You should have received a copy of the GNU General Public License along with | |
15 | * this program; if not, write to the Free Software Foundation, Inc., 59 | |
16 | * Temple Place - Suite 330, Boston, MA 02111-1307, USA. | |
17 | * | |
18 | * The full GNU General Public License is included in this distribution in the | |
19 | * file called COPYING. | |
20 | */ | |
21 | ||
22 | /* | |
23 | * This code implements the DMA subsystem. It provides a HW-neutral interface | |
24 | * for other kernel code to use asynchronous memory copy capabilities, | |
25 | * if present, and allows different HW DMA drivers to register as providing | |
26 | * this capability. | |
27 | * | |
28 | * Due to the fact we are accelerating what is already a relatively fast | |
29 | * operation, the code goes to great lengths to avoid additional overhead, | |
30 | * such as locking. | |
31 | * | |
32 | * LOCKING: | |
33 | * | |
34 | * The subsystem keeps two global lists, dma_device_list and dma_client_list. | |
35 | * Both of these are protected by a mutex, dma_list_mutex. | |
36 | * | |
37 | * Each device has a channels list, which runs unlocked but is never modified | |
38 | * once the device is registered, it's just setup by the driver. | |
39 | * | |
d379b01e DW |
40 | * Each client is responsible for keeping track of the channels it uses. See |
41 | * the definition of dma_event_callback in dmaengine.h. | |
c13c8260 CL |
42 | * |
43 | * Each device has a kref, which is initialized to 1 when the device is | |
891f78ea | 44 | * registered. A kref_get is done for each device registered. When the |
8a5703f8 | 45 | * device is released, the corresponding kref_put is done in the release |
c13c8260 | 46 | * method. Every time one of the device's channels is allocated to a client, |
8a5703f8 | 47 | * a kref_get occurs. When the channel is freed, the corresponding kref_put |
c13c8260 | 48 | * happens. The device's release function does a completion, so |
891f78ea | 49 | * unregister_device does a remove event, device_unregister, a kref_put |
c13c8260 CL |
50 | * for the first reference, then waits on the completion for all other |
51 | * references to finish. | |
52 | * | |
53 | * Each channel has an open-coded implementation of Rusty Russell's "bigref," | |
d379b01e DW |
54 | * with a kref and a per_cpu local_t. A dma_chan_get is called when a client |
55 | * signals that it wants to use a channel, and dma_chan_put is called when | |
8a5703f8 | 56 | * a channel is removed or a client using it is unregistered. A client can |
d379b01e DW |
57 | * take extra references per outstanding transaction, as is the case with |
58 | * the NET DMA client. The release function does a kref_put on the device. | |
59 | * -ChrisL, DanW | |
c13c8260 CL |
60 | */ |
61 | ||
62 | #include <linux/init.h> | |
63 | #include <linux/module.h> | |
7405f74b | 64 | #include <linux/mm.h> |
c13c8260 CL |
65 | #include <linux/device.h> |
66 | #include <linux/dmaengine.h> | |
67 | #include <linux/hardirq.h> | |
68 | #include <linux/spinlock.h> | |
69 | #include <linux/percpu.h> | |
70 | #include <linux/rcupdate.h> | |
71 | #include <linux/mutex.h> | |
7405f74b | 72 | #include <linux/jiffies.h> |
c13c8260 CL |
73 | |
74 | static DEFINE_MUTEX(dma_list_mutex); | |
75 | static LIST_HEAD(dma_device_list); | |
76 | static LIST_HEAD(dma_client_list); | |
77 | ||
78 | /* --- sysfs implementation --- */ | |
79 | ||
891f78ea | 80 | static ssize_t show_memcpy_count(struct device *dev, struct device_attribute *attr, char *buf) |
c13c8260 | 81 | { |
891f78ea | 82 | struct dma_chan *chan = to_dma_chan(dev); |
c13c8260 CL |
83 | unsigned long count = 0; |
84 | int i; | |
85 | ||
17f3ae08 | 86 | for_each_possible_cpu(i) |
c13c8260 CL |
87 | count += per_cpu_ptr(chan->local, i)->memcpy_count; |
88 | ||
89 | return sprintf(buf, "%lu\n", count); | |
90 | } | |
91 | ||
891f78ea TJ |
92 | static ssize_t show_bytes_transferred(struct device *dev, struct device_attribute *attr, |
93 | char *buf) | |
c13c8260 | 94 | { |
891f78ea | 95 | struct dma_chan *chan = to_dma_chan(dev); |
c13c8260 CL |
96 | unsigned long count = 0; |
97 | int i; | |
98 | ||
17f3ae08 | 99 | for_each_possible_cpu(i) |
c13c8260 CL |
100 | count += per_cpu_ptr(chan->local, i)->bytes_transferred; |
101 | ||
102 | return sprintf(buf, "%lu\n", count); | |
103 | } | |
104 | ||
891f78ea | 105 | static ssize_t show_in_use(struct device *dev, struct device_attribute *attr, char *buf) |
c13c8260 | 106 | { |
891f78ea | 107 | struct dma_chan *chan = to_dma_chan(dev); |
d379b01e DW |
108 | int in_use = 0; |
109 | ||
110 | if (unlikely(chan->slow_ref) && | |
111 | atomic_read(&chan->refcount.refcount) > 1) | |
112 | in_use = 1; | |
113 | else { | |
114 | if (local_read(&(per_cpu_ptr(chan->local, | |
115 | get_cpu())->refcount)) > 0) | |
116 | in_use = 1; | |
117 | put_cpu(); | |
118 | } | |
c13c8260 | 119 | |
d379b01e | 120 | return sprintf(buf, "%d\n", in_use); |
c13c8260 CL |
121 | } |
122 | ||
891f78ea | 123 | static struct device_attribute dma_attrs[] = { |
c13c8260 CL |
124 | __ATTR(memcpy_count, S_IRUGO, show_memcpy_count, NULL), |
125 | __ATTR(bytes_transferred, S_IRUGO, show_bytes_transferred, NULL), | |
126 | __ATTR(in_use, S_IRUGO, show_in_use, NULL), | |
127 | __ATTR_NULL | |
128 | }; | |
129 | ||
130 | static void dma_async_device_cleanup(struct kref *kref); | |
131 | ||
891f78ea | 132 | static void dma_dev_release(struct device *dev) |
c13c8260 | 133 | { |
891f78ea | 134 | struct dma_chan *chan = to_dma_chan(dev); |
c13c8260 CL |
135 | kref_put(&chan->device->refcount, dma_async_device_cleanup); |
136 | } | |
137 | ||
138 | static struct class dma_devclass = { | |
891f78ea TJ |
139 | .name = "dma", |
140 | .dev_attrs = dma_attrs, | |
141 | .dev_release = dma_dev_release, | |
c13c8260 CL |
142 | }; |
143 | ||
144 | /* --- client and device registration --- */ | |
145 | ||
d379b01e DW |
146 | #define dma_chan_satisfies_mask(chan, mask) \ |
147 | __dma_chan_satisfies_mask((chan), &(mask)) | |
148 | static int | |
149 | __dma_chan_satisfies_mask(struct dma_chan *chan, dma_cap_mask_t *want) | |
150 | { | |
151 | dma_cap_mask_t has; | |
152 | ||
153 | bitmap_and(has.bits, want->bits, chan->device->cap_mask.bits, | |
154 | DMA_TX_TYPE_END); | |
155 | return bitmap_equal(want->bits, has.bits, DMA_TX_TYPE_END); | |
156 | } | |
157 | ||
c13c8260 | 158 | /** |
d379b01e | 159 | * dma_client_chan_alloc - try to allocate channels to a client |
c13c8260 CL |
160 | * @client: &dma_client |
161 | * | |
162 | * Called with dma_list_mutex held. | |
163 | */ | |
d379b01e | 164 | static void dma_client_chan_alloc(struct dma_client *client) |
c13c8260 CL |
165 | { |
166 | struct dma_device *device; | |
167 | struct dma_chan *chan; | |
c13c8260 | 168 | int desc; /* allocated descriptor count */ |
d379b01e | 169 | enum dma_state_client ack; |
c13c8260 | 170 | |
d379b01e | 171 | /* Find a channel */ |
dc0ee643 HS |
172 | list_for_each_entry(device, &dma_device_list, global_node) { |
173 | /* Does the client require a specific DMA controller? */ | |
174 | if (client->slave && client->slave->dma_dev | |
175 | && client->slave->dma_dev != device->dev) | |
176 | continue; | |
177 | ||
c13c8260 | 178 | list_for_each_entry(chan, &device->channels, device_node) { |
d379b01e | 179 | if (!dma_chan_satisfies_mask(chan, client->cap_mask)) |
c13c8260 CL |
180 | continue; |
181 | ||
848c536a HS |
182 | desc = chan->device->device_alloc_chan_resources( |
183 | chan, client); | |
c13c8260 | 184 | if (desc >= 0) { |
d379b01e DW |
185 | ack = client->event_callback(client, |
186 | chan, | |
187 | DMA_RESOURCE_AVAILABLE); | |
188 | ||
189 | /* we are done once this client rejects | |
190 | * an available resource | |
191 | */ | |
7cc5bf9a | 192 | if (ack == DMA_ACK) { |
d379b01e | 193 | dma_chan_get(chan); |
7cc5bf9a DW |
194 | chan->client_count++; |
195 | } else if (ack == DMA_NAK) | |
d379b01e | 196 | return; |
c13c8260 CL |
197 | } |
198 | } | |
dc0ee643 | 199 | } |
c13c8260 CL |
200 | } |
201 | ||
7405f74b DW |
202 | enum dma_status dma_sync_wait(struct dma_chan *chan, dma_cookie_t cookie) |
203 | { | |
204 | enum dma_status status; | |
205 | unsigned long dma_sync_wait_timeout = jiffies + msecs_to_jiffies(5000); | |
206 | ||
207 | dma_async_issue_pending(chan); | |
208 | do { | |
209 | status = dma_async_is_tx_complete(chan, cookie, NULL, NULL); | |
210 | if (time_after_eq(jiffies, dma_sync_wait_timeout)) { | |
211 | printk(KERN_ERR "dma_sync_wait_timeout!\n"); | |
212 | return DMA_ERROR; | |
213 | } | |
214 | } while (status == DMA_IN_PROGRESS); | |
215 | ||
216 | return status; | |
217 | } | |
218 | EXPORT_SYMBOL(dma_sync_wait); | |
219 | ||
c13c8260 | 220 | /** |
6508871e RD |
221 | * dma_chan_cleanup - release a DMA channel's resources |
222 | * @kref: kernel reference structure that contains the DMA channel device | |
c13c8260 CL |
223 | */ |
224 | void dma_chan_cleanup(struct kref *kref) | |
225 | { | |
226 | struct dma_chan *chan = container_of(kref, struct dma_chan, refcount); | |
227 | chan->device->device_free_chan_resources(chan); | |
c13c8260 CL |
228 | kref_put(&chan->device->refcount, dma_async_device_cleanup); |
229 | } | |
765e3d8a | 230 | EXPORT_SYMBOL(dma_chan_cleanup); |
c13c8260 CL |
231 | |
232 | static void dma_chan_free_rcu(struct rcu_head *rcu) | |
233 | { | |
234 | struct dma_chan *chan = container_of(rcu, struct dma_chan, rcu); | |
235 | int bias = 0x7FFFFFFF; | |
236 | int i; | |
17f3ae08 | 237 | for_each_possible_cpu(i) |
c13c8260 CL |
238 | bias -= local_read(&per_cpu_ptr(chan->local, i)->refcount); |
239 | atomic_sub(bias, &chan->refcount.refcount); | |
240 | kref_put(&chan->refcount, dma_chan_cleanup); | |
241 | } | |
242 | ||
d379b01e | 243 | static void dma_chan_release(struct dma_chan *chan) |
c13c8260 CL |
244 | { |
245 | atomic_add(0x7FFFFFFF, &chan->refcount.refcount); | |
246 | chan->slow_ref = 1; | |
247 | call_rcu(&chan->rcu, dma_chan_free_rcu); | |
248 | } | |
249 | ||
250 | /** | |
d379b01e | 251 | * dma_chans_notify_available - broadcast available channels to the clients |
c13c8260 | 252 | */ |
d379b01e | 253 | static void dma_clients_notify_available(void) |
c13c8260 CL |
254 | { |
255 | struct dma_client *client; | |
c13c8260 CL |
256 | |
257 | mutex_lock(&dma_list_mutex); | |
258 | ||
d379b01e DW |
259 | list_for_each_entry(client, &dma_client_list, global_node) |
260 | dma_client_chan_alloc(client); | |
c13c8260 CL |
261 | |
262 | mutex_unlock(&dma_list_mutex); | |
263 | } | |
264 | ||
265 | /** | |
d379b01e DW |
266 | * dma_chans_notify_available - tell the clients that a channel is going away |
267 | * @chan: channel on its way out | |
c13c8260 | 268 | */ |
d379b01e | 269 | static void dma_clients_notify_removed(struct dma_chan *chan) |
c13c8260 CL |
270 | { |
271 | struct dma_client *client; | |
d379b01e | 272 | enum dma_state_client ack; |
c13c8260 | 273 | |
d379b01e DW |
274 | mutex_lock(&dma_list_mutex); |
275 | ||
276 | list_for_each_entry(client, &dma_client_list, global_node) { | |
277 | ack = client->event_callback(client, chan, | |
278 | DMA_RESOURCE_REMOVED); | |
279 | ||
280 | /* client was holding resources for this channel so | |
281 | * free it | |
282 | */ | |
7cc5bf9a | 283 | if (ack == DMA_ACK) { |
d379b01e | 284 | dma_chan_put(chan); |
7cc5bf9a DW |
285 | chan->client_count--; |
286 | } | |
d379b01e | 287 | } |
c13c8260 | 288 | |
d379b01e DW |
289 | mutex_unlock(&dma_list_mutex); |
290 | } | |
c13c8260 | 291 | |
d379b01e DW |
292 | /** |
293 | * dma_async_client_register - register a &dma_client | |
294 | * @client: ptr to a client structure with valid 'event_callback' and 'cap_mask' | |
295 | */ | |
296 | void dma_async_client_register(struct dma_client *client) | |
297 | { | |
dc0ee643 HS |
298 | /* validate client data */ |
299 | BUG_ON(dma_has_cap(DMA_SLAVE, client->cap_mask) && | |
300 | !client->slave); | |
301 | ||
c13c8260 CL |
302 | mutex_lock(&dma_list_mutex); |
303 | list_add_tail(&client->global_node, &dma_client_list); | |
304 | mutex_unlock(&dma_list_mutex); | |
c13c8260 | 305 | } |
765e3d8a | 306 | EXPORT_SYMBOL(dma_async_client_register); |
c13c8260 CL |
307 | |
308 | /** | |
309 | * dma_async_client_unregister - unregister a client and free the &dma_client | |
6508871e | 310 | * @client: &dma_client to free |
c13c8260 CL |
311 | * |
312 | * Force frees any allocated DMA channels, frees the &dma_client memory | |
313 | */ | |
314 | void dma_async_client_unregister(struct dma_client *client) | |
315 | { | |
d379b01e | 316 | struct dma_device *device; |
c13c8260 | 317 | struct dma_chan *chan; |
d379b01e | 318 | enum dma_state_client ack; |
c13c8260 CL |
319 | |
320 | if (!client) | |
321 | return; | |
322 | ||
c13c8260 | 323 | mutex_lock(&dma_list_mutex); |
d379b01e DW |
324 | /* free all channels the client is holding */ |
325 | list_for_each_entry(device, &dma_device_list, global_node) | |
326 | list_for_each_entry(chan, &device->channels, device_node) { | |
327 | ack = client->event_callback(client, chan, | |
328 | DMA_RESOURCE_REMOVED); | |
329 | ||
7cc5bf9a | 330 | if (ack == DMA_ACK) { |
d379b01e | 331 | dma_chan_put(chan); |
7cc5bf9a DW |
332 | chan->client_count--; |
333 | } | |
d379b01e DW |
334 | } |
335 | ||
c13c8260 CL |
336 | list_del(&client->global_node); |
337 | mutex_unlock(&dma_list_mutex); | |
c13c8260 | 338 | } |
765e3d8a | 339 | EXPORT_SYMBOL(dma_async_client_unregister); |
c13c8260 CL |
340 | |
341 | /** | |
d379b01e DW |
342 | * dma_async_client_chan_request - send all available channels to the |
343 | * client that satisfy the capability mask | |
344 | * @client - requester | |
c13c8260 | 345 | */ |
d379b01e | 346 | void dma_async_client_chan_request(struct dma_client *client) |
c13c8260 | 347 | { |
d379b01e DW |
348 | mutex_lock(&dma_list_mutex); |
349 | dma_client_chan_alloc(client); | |
350 | mutex_unlock(&dma_list_mutex); | |
c13c8260 | 351 | } |
765e3d8a | 352 | EXPORT_SYMBOL(dma_async_client_chan_request); |
c13c8260 CL |
353 | |
354 | /** | |
6508871e | 355 | * dma_async_device_register - registers DMA devices found |
c13c8260 CL |
356 | * @device: &dma_device |
357 | */ | |
358 | int dma_async_device_register(struct dma_device *device) | |
359 | { | |
360 | static int id; | |
ff487fb7 | 361 | int chancnt = 0, rc; |
c13c8260 CL |
362 | struct dma_chan* chan; |
363 | ||
364 | if (!device) | |
365 | return -ENODEV; | |
366 | ||
7405f74b DW |
367 | /* validate device routines */ |
368 | BUG_ON(dma_has_cap(DMA_MEMCPY, device->cap_mask) && | |
369 | !device->device_prep_dma_memcpy); | |
370 | BUG_ON(dma_has_cap(DMA_XOR, device->cap_mask) && | |
371 | !device->device_prep_dma_xor); | |
372 | BUG_ON(dma_has_cap(DMA_ZERO_SUM, device->cap_mask) && | |
373 | !device->device_prep_dma_zero_sum); | |
374 | BUG_ON(dma_has_cap(DMA_MEMSET, device->cap_mask) && | |
375 | !device->device_prep_dma_memset); | |
9b941c66 | 376 | BUG_ON(dma_has_cap(DMA_INTERRUPT, device->cap_mask) && |
7405f74b | 377 | !device->device_prep_dma_interrupt); |
dc0ee643 HS |
378 | BUG_ON(dma_has_cap(DMA_SLAVE, device->cap_mask) && |
379 | !device->device_prep_slave_sg); | |
380 | BUG_ON(dma_has_cap(DMA_SLAVE, device->cap_mask) && | |
381 | !device->device_terminate_all); | |
7405f74b DW |
382 | |
383 | BUG_ON(!device->device_alloc_chan_resources); | |
384 | BUG_ON(!device->device_free_chan_resources); | |
7405f74b DW |
385 | BUG_ON(!device->device_is_tx_complete); |
386 | BUG_ON(!device->device_issue_pending); | |
387 | BUG_ON(!device->dev); | |
388 | ||
c13c8260 CL |
389 | init_completion(&device->done); |
390 | kref_init(&device->refcount); | |
b0b42b16 DW |
391 | |
392 | mutex_lock(&dma_list_mutex); | |
c13c8260 | 393 | device->dev_id = id++; |
b0b42b16 | 394 | mutex_unlock(&dma_list_mutex); |
c13c8260 CL |
395 | |
396 | /* represent channels in sysfs. Probably want devs too */ | |
397 | list_for_each_entry(chan, &device->channels, device_node) { | |
398 | chan->local = alloc_percpu(typeof(*chan->local)); | |
399 | if (chan->local == NULL) | |
400 | continue; | |
401 | ||
402 | chan->chan_id = chancnt++; | |
891f78ea | 403 | chan->dev.class = &dma_devclass; |
1099dc79 | 404 | chan->dev.parent = device->dev; |
06190d84 KS |
405 | dev_set_name(&chan->dev, "dma%dchan%d", |
406 | device->dev_id, chan->chan_id); | |
c13c8260 | 407 | |
891f78ea | 408 | rc = device_register(&chan->dev); |
ff487fb7 JG |
409 | if (rc) { |
410 | chancnt--; | |
411 | free_percpu(chan->local); | |
412 | chan->local = NULL; | |
413 | goto err_out; | |
414 | } | |
415 | ||
348badf1 HS |
416 | /* One for the channel, one of the class device */ |
417 | kref_get(&device->refcount); | |
c13c8260 | 418 | kref_get(&device->refcount); |
d379b01e | 419 | kref_init(&chan->refcount); |
7cc5bf9a | 420 | chan->client_count = 0; |
d379b01e DW |
421 | chan->slow_ref = 0; |
422 | INIT_RCU_HEAD(&chan->rcu); | |
c13c8260 CL |
423 | } |
424 | ||
425 | mutex_lock(&dma_list_mutex); | |
426 | list_add_tail(&device->global_node, &dma_device_list); | |
427 | mutex_unlock(&dma_list_mutex); | |
428 | ||
d379b01e | 429 | dma_clients_notify_available(); |
c13c8260 CL |
430 | |
431 | return 0; | |
ff487fb7 JG |
432 | |
433 | err_out: | |
434 | list_for_each_entry(chan, &device->channels, device_node) { | |
435 | if (chan->local == NULL) | |
436 | continue; | |
437 | kref_put(&device->refcount, dma_async_device_cleanup); | |
891f78ea | 438 | device_unregister(&chan->dev); |
ff487fb7 JG |
439 | chancnt--; |
440 | free_percpu(chan->local); | |
441 | } | |
442 | return rc; | |
c13c8260 | 443 | } |
765e3d8a | 444 | EXPORT_SYMBOL(dma_async_device_register); |
c13c8260 CL |
445 | |
446 | /** | |
6508871e RD |
447 | * dma_async_device_cleanup - function called when all references are released |
448 | * @kref: kernel reference object | |
c13c8260 CL |
449 | */ |
450 | static void dma_async_device_cleanup(struct kref *kref) | |
451 | { | |
452 | struct dma_device *device; | |
453 | ||
454 | device = container_of(kref, struct dma_device, refcount); | |
455 | complete(&device->done); | |
456 | } | |
457 | ||
6508871e RD |
458 | /** |
459 | * dma_async_device_unregister - unregisters DMA devices | |
460 | * @device: &dma_device | |
461 | */ | |
462 | void dma_async_device_unregister(struct dma_device *device) | |
c13c8260 CL |
463 | { |
464 | struct dma_chan *chan; | |
c13c8260 CL |
465 | |
466 | mutex_lock(&dma_list_mutex); | |
467 | list_del(&device->global_node); | |
468 | mutex_unlock(&dma_list_mutex); | |
469 | ||
470 | list_for_each_entry(chan, &device->channels, device_node) { | |
d379b01e | 471 | dma_clients_notify_removed(chan); |
891f78ea | 472 | device_unregister(&chan->dev); |
d379b01e | 473 | dma_chan_release(chan); |
c13c8260 | 474 | } |
c13c8260 CL |
475 | |
476 | kref_put(&device->refcount, dma_async_device_cleanup); | |
477 | wait_for_completion(&device->done); | |
478 | } | |
765e3d8a | 479 | EXPORT_SYMBOL(dma_async_device_unregister); |
c13c8260 | 480 | |
7405f74b DW |
481 | /** |
482 | * dma_async_memcpy_buf_to_buf - offloaded copy between virtual addresses | |
483 | * @chan: DMA channel to offload copy to | |
484 | * @dest: destination address (virtual) | |
485 | * @src: source address (virtual) | |
486 | * @len: length | |
487 | * | |
488 | * Both @dest and @src must be mappable to a bus address according to the | |
489 | * DMA mapping API rules for streaming mappings. | |
490 | * Both @dest and @src must stay memory resident (kernel memory or locked | |
491 | * user space pages). | |
492 | */ | |
493 | dma_cookie_t | |
494 | dma_async_memcpy_buf_to_buf(struct dma_chan *chan, void *dest, | |
495 | void *src, size_t len) | |
496 | { | |
497 | struct dma_device *dev = chan->device; | |
498 | struct dma_async_tx_descriptor *tx; | |
0036731c | 499 | dma_addr_t dma_dest, dma_src; |
7405f74b DW |
500 | dma_cookie_t cookie; |
501 | int cpu; | |
502 | ||
0036731c DW |
503 | dma_src = dma_map_single(dev->dev, src, len, DMA_TO_DEVICE); |
504 | dma_dest = dma_map_single(dev->dev, dest, len, DMA_FROM_DEVICE); | |
636bdeaa DW |
505 | tx = dev->device_prep_dma_memcpy(chan, dma_dest, dma_src, len, |
506 | DMA_CTRL_ACK); | |
0036731c DW |
507 | |
508 | if (!tx) { | |
509 | dma_unmap_single(dev->dev, dma_src, len, DMA_TO_DEVICE); | |
510 | dma_unmap_single(dev->dev, dma_dest, len, DMA_FROM_DEVICE); | |
7405f74b | 511 | return -ENOMEM; |
0036731c | 512 | } |
7405f74b | 513 | |
7405f74b | 514 | tx->callback = NULL; |
7405f74b DW |
515 | cookie = tx->tx_submit(tx); |
516 | ||
517 | cpu = get_cpu(); | |
518 | per_cpu_ptr(chan->local, cpu)->bytes_transferred += len; | |
519 | per_cpu_ptr(chan->local, cpu)->memcpy_count++; | |
520 | put_cpu(); | |
521 | ||
522 | return cookie; | |
523 | } | |
524 | EXPORT_SYMBOL(dma_async_memcpy_buf_to_buf); | |
525 | ||
526 | /** | |
527 | * dma_async_memcpy_buf_to_pg - offloaded copy from address to page | |
528 | * @chan: DMA channel to offload copy to | |
529 | * @page: destination page | |
530 | * @offset: offset in page to copy to | |
531 | * @kdata: source address (virtual) | |
532 | * @len: length | |
533 | * | |
534 | * Both @page/@offset and @kdata must be mappable to a bus address according | |
535 | * to the DMA mapping API rules for streaming mappings. | |
536 | * Both @page/@offset and @kdata must stay memory resident (kernel memory or | |
537 | * locked user space pages) | |
538 | */ | |
539 | dma_cookie_t | |
540 | dma_async_memcpy_buf_to_pg(struct dma_chan *chan, struct page *page, | |
541 | unsigned int offset, void *kdata, size_t len) | |
542 | { | |
543 | struct dma_device *dev = chan->device; | |
544 | struct dma_async_tx_descriptor *tx; | |
0036731c | 545 | dma_addr_t dma_dest, dma_src; |
7405f74b DW |
546 | dma_cookie_t cookie; |
547 | int cpu; | |
548 | ||
0036731c DW |
549 | dma_src = dma_map_single(dev->dev, kdata, len, DMA_TO_DEVICE); |
550 | dma_dest = dma_map_page(dev->dev, page, offset, len, DMA_FROM_DEVICE); | |
636bdeaa DW |
551 | tx = dev->device_prep_dma_memcpy(chan, dma_dest, dma_src, len, |
552 | DMA_CTRL_ACK); | |
0036731c DW |
553 | |
554 | if (!tx) { | |
555 | dma_unmap_single(dev->dev, dma_src, len, DMA_TO_DEVICE); | |
556 | dma_unmap_page(dev->dev, dma_dest, len, DMA_FROM_DEVICE); | |
7405f74b | 557 | return -ENOMEM; |
0036731c | 558 | } |
7405f74b | 559 | |
7405f74b | 560 | tx->callback = NULL; |
7405f74b DW |
561 | cookie = tx->tx_submit(tx); |
562 | ||
563 | cpu = get_cpu(); | |
564 | per_cpu_ptr(chan->local, cpu)->bytes_transferred += len; | |
565 | per_cpu_ptr(chan->local, cpu)->memcpy_count++; | |
566 | put_cpu(); | |
567 | ||
568 | return cookie; | |
569 | } | |
570 | EXPORT_SYMBOL(dma_async_memcpy_buf_to_pg); | |
571 | ||
572 | /** | |
573 | * dma_async_memcpy_pg_to_pg - offloaded copy from page to page | |
574 | * @chan: DMA channel to offload copy to | |
575 | * @dest_pg: destination page | |
576 | * @dest_off: offset in page to copy to | |
577 | * @src_pg: source page | |
578 | * @src_off: offset in page to copy from | |
579 | * @len: length | |
580 | * | |
581 | * Both @dest_page/@dest_off and @src_page/@src_off must be mappable to a bus | |
582 | * address according to the DMA mapping API rules for streaming mappings. | |
583 | * Both @dest_page/@dest_off and @src_page/@src_off must stay memory resident | |
584 | * (kernel memory or locked user space pages). | |
585 | */ | |
586 | dma_cookie_t | |
587 | dma_async_memcpy_pg_to_pg(struct dma_chan *chan, struct page *dest_pg, | |
588 | unsigned int dest_off, struct page *src_pg, unsigned int src_off, | |
589 | size_t len) | |
590 | { | |
591 | struct dma_device *dev = chan->device; | |
592 | struct dma_async_tx_descriptor *tx; | |
0036731c | 593 | dma_addr_t dma_dest, dma_src; |
7405f74b DW |
594 | dma_cookie_t cookie; |
595 | int cpu; | |
596 | ||
0036731c DW |
597 | dma_src = dma_map_page(dev->dev, src_pg, src_off, len, DMA_TO_DEVICE); |
598 | dma_dest = dma_map_page(dev->dev, dest_pg, dest_off, len, | |
599 | DMA_FROM_DEVICE); | |
636bdeaa DW |
600 | tx = dev->device_prep_dma_memcpy(chan, dma_dest, dma_src, len, |
601 | DMA_CTRL_ACK); | |
0036731c DW |
602 | |
603 | if (!tx) { | |
604 | dma_unmap_page(dev->dev, dma_src, len, DMA_TO_DEVICE); | |
605 | dma_unmap_page(dev->dev, dma_dest, len, DMA_FROM_DEVICE); | |
7405f74b | 606 | return -ENOMEM; |
0036731c | 607 | } |
7405f74b | 608 | |
7405f74b | 609 | tx->callback = NULL; |
7405f74b DW |
610 | cookie = tx->tx_submit(tx); |
611 | ||
612 | cpu = get_cpu(); | |
613 | per_cpu_ptr(chan->local, cpu)->bytes_transferred += len; | |
614 | per_cpu_ptr(chan->local, cpu)->memcpy_count++; | |
615 | put_cpu(); | |
616 | ||
617 | return cookie; | |
618 | } | |
619 | EXPORT_SYMBOL(dma_async_memcpy_pg_to_pg); | |
620 | ||
621 | void dma_async_tx_descriptor_init(struct dma_async_tx_descriptor *tx, | |
622 | struct dma_chan *chan) | |
623 | { | |
624 | tx->chan = chan; | |
625 | spin_lock_init(&tx->lock); | |
7405f74b DW |
626 | } |
627 | EXPORT_SYMBOL(dma_async_tx_descriptor_init); | |
628 | ||
c13c8260 CL |
629 | static int __init dma_bus_init(void) |
630 | { | |
631 | mutex_init(&dma_list_mutex); | |
632 | return class_register(&dma_devclass); | |
633 | } | |
c13c8260 CL |
634 | subsys_initcall(dma_bus_init); |
635 |