]>
Commit | Line | Data |
---|---|---|
0bbd5f4e CL |
1 | /* |
2 | * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved. | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or modify it | |
5 | * under the terms of the GNU General Public License as published by the Free | |
6 | * Software Foundation; either version 2 of the License, or (at your option) | |
7 | * any later version. | |
8 | * | |
9 | * This program is distributed in the hope that it will be useful, but WITHOUT | |
10 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
11 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | |
12 | * more details. | |
13 | * | |
14 | * You should have received a copy of the GNU General Public License along with | |
15 | * this program; if not, write to the Free Software Foundation, Inc., 59 | |
16 | * Temple Place - Suite 330, Boston, MA 02111-1307, USA. | |
17 | * | |
18 | * The full GNU General Public License is included in this distribution in the | |
19 | * file called COPYING. | |
20 | */ | |
21 | ||
22 | /* | |
23 | * This driver supports an Intel I/OAT DMA engine, which does asynchronous | |
24 | * copy operations. | |
25 | */ | |
26 | ||
27 | #include <linux/init.h> | |
28 | #include <linux/module.h> | |
29 | #include <linux/pci.h> | |
30 | #include <linux/interrupt.h> | |
31 | #include <linux/dmaengine.h> | |
32 | #include <linux/delay.h> | |
6b00c92c | 33 | #include <linux/dma-mapping.h> |
0bbd5f4e CL |
34 | #include "ioatdma.h" |
35 | #include "ioatdma_io.h" | |
36 | #include "ioatdma_registers.h" | |
37 | #include "ioatdma_hw.h" | |
38 | ||
39 | #define to_ioat_chan(chan) container_of(chan, struct ioat_dma_chan, common) | |
40 | #define to_ioat_device(dev) container_of(dev, struct ioat_device, common) | |
41 | #define to_ioat_desc(lh) container_of(lh, struct ioat_desc_sw, node) | |
42 | ||
43 | /* internal functions */ | |
44 | static int __devinit ioat_probe(struct pci_dev *pdev, const struct pci_device_id *ent); | |
45 | static void __devexit ioat_remove(struct pci_dev *pdev); | |
46 | ||
47 | static int enumerate_dma_channels(struct ioat_device *device) | |
48 | { | |
49 | u8 xfercap_scale; | |
50 | u32 xfercap; | |
51 | int i; | |
52 | struct ioat_dma_chan *ioat_chan; | |
53 | ||
54 | device->common.chancnt = ioatdma_read8(device, IOAT_CHANCNT_OFFSET); | |
55 | xfercap_scale = ioatdma_read8(device, IOAT_XFERCAP_OFFSET); | |
56 | xfercap = (xfercap_scale == 0 ? -1 : (1UL << xfercap_scale)); | |
57 | ||
58 | for (i = 0; i < device->common.chancnt; i++) { | |
59 | ioat_chan = kzalloc(sizeof(*ioat_chan), GFP_KERNEL); | |
60 | if (!ioat_chan) { | |
61 | device->common.chancnt = i; | |
62 | break; | |
63 | } | |
64 | ||
65 | ioat_chan->device = device; | |
66 | ioat_chan->reg_base = device->reg_base + (0x80 * (i + 1)); | |
67 | ioat_chan->xfercap = xfercap; | |
68 | spin_lock_init(&ioat_chan->cleanup_lock); | |
69 | spin_lock_init(&ioat_chan->desc_lock); | |
70 | INIT_LIST_HEAD(&ioat_chan->free_desc); | |
71 | INIT_LIST_HEAD(&ioat_chan->used_desc); | |
72 | /* This should be made common somewhere in dmaengine.c */ | |
73 | ioat_chan->common.device = &device->common; | |
74 | ioat_chan->common.client = NULL; | |
75 | list_add_tail(&ioat_chan->common.device_node, | |
76 | &device->common.channels); | |
77 | } | |
78 | return device->common.chancnt; | |
79 | } | |
80 | ||
81 | static struct ioat_desc_sw *ioat_dma_alloc_descriptor( | |
82 | struct ioat_dma_chan *ioat_chan, | |
83 | int flags) | |
84 | { | |
85 | struct ioat_dma_descriptor *desc; | |
86 | struct ioat_desc_sw *desc_sw; | |
87 | struct ioat_device *ioat_device; | |
88 | dma_addr_t phys; | |
89 | ||
90 | ioat_device = to_ioat_device(ioat_chan->common.device); | |
91 | desc = pci_pool_alloc(ioat_device->dma_pool, flags, &phys); | |
92 | if (unlikely(!desc)) | |
93 | return NULL; | |
94 | ||
95 | desc_sw = kzalloc(sizeof(*desc_sw), flags); | |
96 | if (unlikely(!desc_sw)) { | |
97 | pci_pool_free(ioat_device->dma_pool, desc, phys); | |
98 | return NULL; | |
99 | } | |
100 | ||
101 | memset(desc, 0, sizeof(*desc)); | |
102 | desc_sw->hw = desc; | |
103 | desc_sw->phys = phys; | |
104 | ||
105 | return desc_sw; | |
106 | } | |
107 | ||
108 | #define INITIAL_IOAT_DESC_COUNT 128 | |
109 | ||
110 | static void ioat_start_null_desc(struct ioat_dma_chan *ioat_chan); | |
111 | ||
112 | /* returns the actual number of allocated descriptors */ | |
113 | static int ioat_dma_alloc_chan_resources(struct dma_chan *chan) | |
114 | { | |
115 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan); | |
116 | struct ioat_desc_sw *desc = NULL; | |
117 | u16 chanctrl; | |
118 | u32 chanerr; | |
119 | int i; | |
120 | LIST_HEAD(tmp_list); | |
121 | ||
122 | /* | |
123 | * In-use bit automatically set by reading chanctrl | |
124 | * If 0, we got it, if 1, someone else did | |
125 | */ | |
126 | chanctrl = ioatdma_chan_read16(ioat_chan, IOAT_CHANCTRL_OFFSET); | |
127 | if (chanctrl & IOAT_CHANCTRL_CHANNEL_IN_USE) | |
128 | return -EBUSY; | |
129 | ||
130 | /* Setup register to interrupt and write completion status on error */ | |
131 | chanctrl = IOAT_CHANCTRL_CHANNEL_IN_USE | | |
132 | IOAT_CHANCTRL_ERR_INT_EN | | |
133 | IOAT_CHANCTRL_ANY_ERR_ABORT_EN | | |
134 | IOAT_CHANCTRL_ERR_COMPLETION_EN; | |
135 | ioatdma_chan_write16(ioat_chan, IOAT_CHANCTRL_OFFSET, chanctrl); | |
136 | ||
137 | chanerr = ioatdma_chan_read32(ioat_chan, IOAT_CHANERR_OFFSET); | |
138 | if (chanerr) { | |
139 | printk("IOAT: CHANERR = %x, clearing\n", chanerr); | |
140 | ioatdma_chan_write32(ioat_chan, IOAT_CHANERR_OFFSET, chanerr); | |
141 | } | |
142 | ||
143 | /* Allocate descriptors */ | |
144 | for (i = 0; i < INITIAL_IOAT_DESC_COUNT; i++) { | |
145 | desc = ioat_dma_alloc_descriptor(ioat_chan, GFP_KERNEL); | |
146 | if (!desc) { | |
147 | printk(KERN_ERR "IOAT: Only %d initial descriptors\n", i); | |
148 | break; | |
149 | } | |
150 | list_add_tail(&desc->node, &tmp_list); | |
151 | } | |
152 | spin_lock_bh(&ioat_chan->desc_lock); | |
153 | list_splice(&tmp_list, &ioat_chan->free_desc); | |
154 | spin_unlock_bh(&ioat_chan->desc_lock); | |
155 | ||
156 | /* allocate a completion writeback area */ | |
157 | /* doing 2 32bit writes to mmio since 1 64b write doesn't work */ | |
158 | ioat_chan->completion_virt = | |
159 | pci_pool_alloc(ioat_chan->device->completion_pool, | |
160 | GFP_KERNEL, | |
161 | &ioat_chan->completion_addr); | |
162 | memset(ioat_chan->completion_virt, 0, | |
163 | sizeof(*ioat_chan->completion_virt)); | |
164 | ioatdma_chan_write32(ioat_chan, IOAT_CHANCMP_OFFSET_LOW, | |
165 | ((u64) ioat_chan->completion_addr) & 0x00000000FFFFFFFF); | |
166 | ioatdma_chan_write32(ioat_chan, IOAT_CHANCMP_OFFSET_HIGH, | |
167 | ((u64) ioat_chan->completion_addr) >> 32); | |
168 | ||
169 | ioat_start_null_desc(ioat_chan); | |
170 | return i; | |
171 | } | |
172 | ||
173 | static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat_chan); | |
174 | ||
175 | static void ioat_dma_free_chan_resources(struct dma_chan *chan) | |
176 | { | |
177 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan); | |
178 | struct ioat_device *ioat_device = to_ioat_device(chan->device); | |
179 | struct ioat_desc_sw *desc, *_desc; | |
180 | u16 chanctrl; | |
181 | int in_use_descs = 0; | |
182 | ||
183 | ioat_dma_memcpy_cleanup(ioat_chan); | |
184 | ||
185 | ioatdma_chan_write8(ioat_chan, IOAT_CHANCMD_OFFSET, IOAT_CHANCMD_RESET); | |
186 | ||
187 | spin_lock_bh(&ioat_chan->desc_lock); | |
188 | list_for_each_entry_safe(desc, _desc, &ioat_chan->used_desc, node) { | |
189 | in_use_descs++; | |
190 | list_del(&desc->node); | |
191 | pci_pool_free(ioat_device->dma_pool, desc->hw, desc->phys); | |
192 | kfree(desc); | |
193 | } | |
194 | list_for_each_entry_safe(desc, _desc, &ioat_chan->free_desc, node) { | |
195 | list_del(&desc->node); | |
196 | pci_pool_free(ioat_device->dma_pool, desc->hw, desc->phys); | |
197 | kfree(desc); | |
198 | } | |
199 | spin_unlock_bh(&ioat_chan->desc_lock); | |
200 | ||
201 | pci_pool_free(ioat_device->completion_pool, | |
202 | ioat_chan->completion_virt, | |
203 | ioat_chan->completion_addr); | |
204 | ||
205 | /* one is ok since we left it on there on purpose */ | |
206 | if (in_use_descs > 1) | |
207 | printk(KERN_ERR "IOAT: Freeing %d in use descriptors!\n", | |
208 | in_use_descs - 1); | |
209 | ||
210 | ioat_chan->last_completion = ioat_chan->completion_addr = 0; | |
211 | ||
212 | /* Tell hw the chan is free */ | |
213 | chanctrl = ioatdma_chan_read16(ioat_chan, IOAT_CHANCTRL_OFFSET); | |
214 | chanctrl &= ~IOAT_CHANCTRL_CHANNEL_IN_USE; | |
215 | ioatdma_chan_write16(ioat_chan, IOAT_CHANCTRL_OFFSET, chanctrl); | |
216 | } | |
217 | ||
218 | /** | |
219 | * do_ioat_dma_memcpy - actual function that initiates a IOAT DMA transaction | |
220 | * @chan: IOAT DMA channel handle | |
221 | * @dest: DMA destination address | |
222 | * @src: DMA source address | |
223 | * @len: transaction length in bytes | |
224 | */ | |
225 | ||
226 | static dma_cookie_t do_ioat_dma_memcpy(struct ioat_dma_chan *ioat_chan, | |
227 | dma_addr_t dest, | |
228 | dma_addr_t src, | |
229 | size_t len) | |
230 | { | |
231 | struct ioat_desc_sw *first; | |
232 | struct ioat_desc_sw *prev; | |
233 | struct ioat_desc_sw *new; | |
234 | dma_cookie_t cookie; | |
235 | LIST_HEAD(new_chain); | |
236 | u32 copy; | |
237 | size_t orig_len; | |
238 | dma_addr_t orig_src, orig_dst; | |
239 | unsigned int desc_count = 0; | |
240 | unsigned int append = 0; | |
241 | ||
242 | if (!ioat_chan || !dest || !src) | |
243 | return -EFAULT; | |
244 | ||
245 | if (!len) | |
246 | return ioat_chan->common.cookie; | |
247 | ||
248 | orig_len = len; | |
249 | orig_src = src; | |
250 | orig_dst = dest; | |
251 | ||
252 | first = NULL; | |
253 | prev = NULL; | |
254 | ||
255 | spin_lock_bh(&ioat_chan->desc_lock); | |
256 | ||
257 | while (len) { | |
258 | if (!list_empty(&ioat_chan->free_desc)) { | |
259 | new = to_ioat_desc(ioat_chan->free_desc.next); | |
260 | list_del(&new->node); | |
261 | } else { | |
262 | /* try to get another desc */ | |
263 | new = ioat_dma_alloc_descriptor(ioat_chan, GFP_ATOMIC); | |
264 | /* will this ever happen? */ | |
265 | /* TODO add upper limit on these */ | |
266 | BUG_ON(!new); | |
267 | } | |
268 | ||
269 | copy = min((u32) len, ioat_chan->xfercap); | |
270 | ||
271 | new->hw->size = copy; | |
272 | new->hw->ctl = 0; | |
273 | new->hw->src_addr = src; | |
274 | new->hw->dst_addr = dest; | |
275 | new->cookie = 0; | |
276 | ||
277 | /* chain together the physical address list for the HW */ | |
278 | if (!first) | |
279 | first = new; | |
280 | else | |
281 | prev->hw->next = (u64) new->phys; | |
282 | ||
283 | prev = new; | |
284 | ||
285 | len -= copy; | |
286 | dest += copy; | |
287 | src += copy; | |
288 | ||
289 | list_add_tail(&new->node, &new_chain); | |
290 | desc_count++; | |
291 | } | |
292 | new->hw->ctl = IOAT_DMA_DESCRIPTOR_CTL_CP_STS; | |
293 | new->hw->next = 0; | |
294 | ||
295 | /* cookie incr and addition to used_list must be atomic */ | |
296 | ||
297 | cookie = ioat_chan->common.cookie; | |
298 | cookie++; | |
299 | if (cookie < 0) | |
300 | cookie = 1; | |
301 | ioat_chan->common.cookie = new->cookie = cookie; | |
302 | ||
303 | pci_unmap_addr_set(new, src, orig_src); | |
304 | pci_unmap_addr_set(new, dst, orig_dst); | |
305 | pci_unmap_len_set(new, src_len, orig_len); | |
306 | pci_unmap_len_set(new, dst_len, orig_len); | |
307 | ||
308 | /* write address into NextDescriptor field of last desc in chain */ | |
309 | to_ioat_desc(ioat_chan->used_desc.prev)->hw->next = first->phys; | |
310 | list_splice_init(&new_chain, ioat_chan->used_desc.prev); | |
311 | ||
312 | ioat_chan->pending += desc_count; | |
313 | if (ioat_chan->pending >= 20) { | |
314 | append = 1; | |
315 | ioat_chan->pending = 0; | |
316 | } | |
317 | ||
318 | spin_unlock_bh(&ioat_chan->desc_lock); | |
319 | ||
320 | if (append) | |
321 | ioatdma_chan_write8(ioat_chan, | |
322 | IOAT_CHANCMD_OFFSET, | |
323 | IOAT_CHANCMD_APPEND); | |
324 | return cookie; | |
325 | } | |
326 | ||
327 | /** | |
328 | * ioat_dma_memcpy_buf_to_buf - wrapper that takes src & dest bufs | |
329 | * @chan: IOAT DMA channel handle | |
330 | * @dest: DMA destination address | |
331 | * @src: DMA source address | |
332 | * @len: transaction length in bytes | |
333 | */ | |
334 | ||
335 | static dma_cookie_t ioat_dma_memcpy_buf_to_buf(struct dma_chan *chan, | |
336 | void *dest, | |
337 | void *src, | |
338 | size_t len) | |
339 | { | |
340 | dma_addr_t dest_addr; | |
341 | dma_addr_t src_addr; | |
342 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan); | |
343 | ||
344 | dest_addr = pci_map_single(ioat_chan->device->pdev, | |
345 | dest, len, PCI_DMA_FROMDEVICE); | |
346 | src_addr = pci_map_single(ioat_chan->device->pdev, | |
347 | src, len, PCI_DMA_TODEVICE); | |
348 | ||
349 | return do_ioat_dma_memcpy(ioat_chan, dest_addr, src_addr, len); | |
350 | } | |
351 | ||
352 | /** | |
353 | * ioat_dma_memcpy_buf_to_pg - wrapper, copying from a buf to a page | |
354 | * @chan: IOAT DMA channel handle | |
355 | * @page: pointer to the page to copy to | |
356 | * @offset: offset into that page | |
357 | * @src: DMA source address | |
358 | * @len: transaction length in bytes | |
359 | */ | |
360 | ||
361 | static dma_cookie_t ioat_dma_memcpy_buf_to_pg(struct dma_chan *chan, | |
362 | struct page *page, | |
363 | unsigned int offset, | |
364 | void *src, | |
365 | size_t len) | |
366 | { | |
367 | dma_addr_t dest_addr; | |
368 | dma_addr_t src_addr; | |
369 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan); | |
370 | ||
371 | dest_addr = pci_map_page(ioat_chan->device->pdev, | |
372 | page, offset, len, PCI_DMA_FROMDEVICE); | |
373 | src_addr = pci_map_single(ioat_chan->device->pdev, | |
374 | src, len, PCI_DMA_TODEVICE); | |
375 | ||
376 | return do_ioat_dma_memcpy(ioat_chan, dest_addr, src_addr, len); | |
377 | } | |
378 | ||
379 | /** | |
380 | * ioat_dma_memcpy_pg_to_pg - wrapper, copying between two pages | |
381 | * @chan: IOAT DMA channel handle | |
382 | * @dest_pg: pointer to the page to copy to | |
383 | * @dest_off: offset into that page | |
384 | * @src_pg: pointer to the page to copy from | |
385 | * @src_off: offset into that page | |
386 | * @len: transaction length in bytes. This is guaranteed to not make a copy | |
387 | * across a page boundary. | |
388 | */ | |
389 | ||
390 | static dma_cookie_t ioat_dma_memcpy_pg_to_pg(struct dma_chan *chan, | |
391 | struct page *dest_pg, | |
392 | unsigned int dest_off, | |
393 | struct page *src_pg, | |
394 | unsigned int src_off, | |
395 | size_t len) | |
396 | { | |
397 | dma_addr_t dest_addr; | |
398 | dma_addr_t src_addr; | |
399 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan); | |
400 | ||
401 | dest_addr = pci_map_page(ioat_chan->device->pdev, | |
402 | dest_pg, dest_off, len, PCI_DMA_FROMDEVICE); | |
403 | src_addr = pci_map_page(ioat_chan->device->pdev, | |
404 | src_pg, src_off, len, PCI_DMA_TODEVICE); | |
405 | ||
406 | return do_ioat_dma_memcpy(ioat_chan, dest_addr, src_addr, len); | |
407 | } | |
408 | ||
409 | /** | |
410 | * ioat_dma_memcpy_issue_pending - push potentially unrecognoized appended descriptors to hw | |
411 | * @chan: DMA channel handle | |
412 | */ | |
413 | ||
414 | static void ioat_dma_memcpy_issue_pending(struct dma_chan *chan) | |
415 | { | |
416 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan); | |
417 | ||
418 | if (ioat_chan->pending != 0) { | |
419 | ioat_chan->pending = 0; | |
420 | ioatdma_chan_write8(ioat_chan, | |
421 | IOAT_CHANCMD_OFFSET, | |
422 | IOAT_CHANCMD_APPEND); | |
423 | } | |
424 | } | |
425 | ||
426 | static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *chan) | |
427 | { | |
428 | unsigned long phys_complete; | |
429 | struct ioat_desc_sw *desc, *_desc; | |
430 | dma_cookie_t cookie = 0; | |
431 | ||
432 | prefetch(chan->completion_virt); | |
433 | ||
434 | if (!spin_trylock(&chan->cleanup_lock)) | |
435 | return; | |
436 | ||
437 | /* The completion writeback can happen at any time, | |
438 | so reads by the driver need to be atomic operations | |
439 | The descriptor physical addresses are limited to 32-bits | |
440 | when the CPU can only do a 32-bit mov */ | |
441 | ||
442 | #if (BITS_PER_LONG == 64) | |
443 | phys_complete = | |
444 | chan->completion_virt->full & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR; | |
445 | #else | |
446 | phys_complete = chan->completion_virt->low & IOAT_LOW_COMPLETION_MASK; | |
447 | #endif | |
448 | ||
449 | if ((chan->completion_virt->full & IOAT_CHANSTS_DMA_TRANSFER_STATUS) == | |
450 | IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED) { | |
451 | printk("IOAT: Channel halted, chanerr = %x\n", | |
452 | ioatdma_chan_read32(chan, IOAT_CHANERR_OFFSET)); | |
453 | ||
454 | /* TODO do something to salvage the situation */ | |
455 | } | |
456 | ||
457 | if (phys_complete == chan->last_completion) { | |
458 | spin_unlock(&chan->cleanup_lock); | |
459 | return; | |
460 | } | |
461 | ||
462 | spin_lock_bh(&chan->desc_lock); | |
463 | list_for_each_entry_safe(desc, _desc, &chan->used_desc, node) { | |
464 | ||
465 | /* | |
466 | * Incoming DMA requests may use multiple descriptors, due to | |
467 | * exceeding xfercap, perhaps. If so, only the last one will | |
468 | * have a cookie, and require unmapping. | |
469 | */ | |
470 | if (desc->cookie) { | |
471 | cookie = desc->cookie; | |
472 | ||
473 | /* yes we are unmapping both _page and _single alloc'd | |
474 | regions with unmap_page. Is this *really* that bad? | |
475 | */ | |
476 | pci_unmap_page(chan->device->pdev, | |
477 | pci_unmap_addr(desc, dst), | |
478 | pci_unmap_len(desc, dst_len), | |
479 | PCI_DMA_FROMDEVICE); | |
480 | pci_unmap_page(chan->device->pdev, | |
481 | pci_unmap_addr(desc, src), | |
482 | pci_unmap_len(desc, src_len), | |
483 | PCI_DMA_TODEVICE); | |
484 | } | |
485 | ||
486 | if (desc->phys != phys_complete) { | |
487 | /* a completed entry, but not the last, so cleanup */ | |
488 | list_del(&desc->node); | |
489 | list_add_tail(&desc->node, &chan->free_desc); | |
490 | } else { | |
491 | /* last used desc. Do not remove, so we can append from | |
492 | it, but don't look at it next time, either */ | |
493 | desc->cookie = 0; | |
494 | ||
495 | /* TODO check status bits? */ | |
496 | break; | |
497 | } | |
498 | } | |
499 | ||
500 | spin_unlock_bh(&chan->desc_lock); | |
501 | ||
502 | chan->last_completion = phys_complete; | |
503 | if (cookie != 0) | |
504 | chan->completed_cookie = cookie; | |
505 | ||
506 | spin_unlock(&chan->cleanup_lock); | |
507 | } | |
508 | ||
509 | /** | |
510 | * ioat_dma_is_complete - poll the status of a IOAT DMA transaction | |
511 | * @chan: IOAT DMA channel handle | |
512 | * @cookie: DMA transaction identifier | |
513 | */ | |
514 | ||
515 | static enum dma_status ioat_dma_is_complete(struct dma_chan *chan, | |
516 | dma_cookie_t cookie, | |
517 | dma_cookie_t *done, | |
518 | dma_cookie_t *used) | |
519 | { | |
520 | struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan); | |
521 | dma_cookie_t last_used; | |
522 | dma_cookie_t last_complete; | |
523 | enum dma_status ret; | |
524 | ||
525 | last_used = chan->cookie; | |
526 | last_complete = ioat_chan->completed_cookie; | |
527 | ||
528 | if (done) | |
529 | *done= last_complete; | |
530 | if (used) | |
531 | *used = last_used; | |
532 | ||
533 | ret = dma_async_is_complete(cookie, last_complete, last_used); | |
534 | if (ret == DMA_SUCCESS) | |
535 | return ret; | |
536 | ||
537 | ioat_dma_memcpy_cleanup(ioat_chan); | |
538 | ||
539 | last_used = chan->cookie; | |
540 | last_complete = ioat_chan->completed_cookie; | |
541 | ||
542 | if (done) | |
543 | *done= last_complete; | |
544 | if (used) | |
545 | *used = last_used; | |
546 | ||
547 | return dma_async_is_complete(cookie, last_complete, last_used); | |
548 | } | |
549 | ||
550 | /* PCI API */ | |
551 | ||
552 | static struct pci_device_id ioat_pci_tbl[] = { | |
553 | { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT) }, | |
554 | { 0, } | |
555 | }; | |
556 | ||
557 | static struct pci_driver ioat_pci_drv = { | |
558 | .name = "ioatdma", | |
559 | .id_table = ioat_pci_tbl, | |
560 | .probe = ioat_probe, | |
561 | .remove = __devexit_p(ioat_remove), | |
562 | }; | |
563 | ||
564 | static irqreturn_t ioat_do_interrupt(int irq, void *data, struct pt_regs *regs) | |
565 | { | |
566 | struct ioat_device *instance = data; | |
567 | unsigned long attnstatus; | |
568 | u8 intrctrl; | |
569 | ||
570 | intrctrl = ioatdma_read8(instance, IOAT_INTRCTRL_OFFSET); | |
571 | ||
572 | if (!(intrctrl & IOAT_INTRCTRL_MASTER_INT_EN)) | |
573 | return IRQ_NONE; | |
574 | ||
575 | if (!(intrctrl & IOAT_INTRCTRL_INT_STATUS)) { | |
576 | ioatdma_write8(instance, IOAT_INTRCTRL_OFFSET, intrctrl); | |
577 | return IRQ_NONE; | |
578 | } | |
579 | ||
580 | attnstatus = ioatdma_read32(instance, IOAT_ATTNSTATUS_OFFSET); | |
581 | ||
582 | printk(KERN_ERR "ioatdma error: interrupt! status %lx\n", attnstatus); | |
583 | ||
584 | ioatdma_write8(instance, IOAT_INTRCTRL_OFFSET, intrctrl); | |
585 | return IRQ_HANDLED; | |
586 | } | |
587 | ||
588 | static void ioat_start_null_desc(struct ioat_dma_chan *ioat_chan) | |
589 | { | |
590 | struct ioat_desc_sw *desc; | |
591 | ||
592 | spin_lock_bh(&ioat_chan->desc_lock); | |
593 | ||
594 | if (!list_empty(&ioat_chan->free_desc)) { | |
595 | desc = to_ioat_desc(ioat_chan->free_desc.next); | |
596 | list_del(&desc->node); | |
597 | } else { | |
598 | /* try to get another desc */ | |
599 | spin_unlock_bh(&ioat_chan->desc_lock); | |
600 | desc = ioat_dma_alloc_descriptor(ioat_chan, GFP_KERNEL); | |
601 | spin_lock_bh(&ioat_chan->desc_lock); | |
602 | /* will this ever happen? */ | |
603 | BUG_ON(!desc); | |
604 | } | |
605 | ||
606 | desc->hw->ctl = IOAT_DMA_DESCRIPTOR_NUL; | |
607 | desc->hw->next = 0; | |
608 | ||
609 | list_add_tail(&desc->node, &ioat_chan->used_desc); | |
610 | spin_unlock_bh(&ioat_chan->desc_lock); | |
611 | ||
612 | #if (BITS_PER_LONG == 64) | |
613 | ioatdma_chan_write64(ioat_chan, IOAT_CHAINADDR_OFFSET, desc->phys); | |
614 | #else | |
615 | ioatdma_chan_write32(ioat_chan, | |
616 | IOAT_CHAINADDR_OFFSET_LOW, | |
617 | (u32) desc->phys); | |
618 | ioatdma_chan_write32(ioat_chan, IOAT_CHAINADDR_OFFSET_HIGH, 0); | |
619 | #endif | |
620 | ioatdma_chan_write8(ioat_chan, IOAT_CHANCMD_OFFSET, IOAT_CHANCMD_START); | |
621 | } | |
622 | ||
623 | /* | |
624 | * Perform a IOAT transaction to verify the HW works. | |
625 | */ | |
626 | #define IOAT_TEST_SIZE 2000 | |
627 | ||
628 | static int ioat_self_test(struct ioat_device *device) | |
629 | { | |
630 | int i; | |
631 | u8 *src; | |
632 | u8 *dest; | |
633 | struct dma_chan *dma_chan; | |
634 | dma_cookie_t cookie; | |
635 | int err = 0; | |
636 | ||
637 | src = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, SLAB_KERNEL); | |
638 | if (!src) | |
639 | return -ENOMEM; | |
640 | dest = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, SLAB_KERNEL); | |
641 | if (!dest) { | |
642 | kfree(src); | |
643 | return -ENOMEM; | |
644 | } | |
645 | ||
646 | /* Fill in src buffer */ | |
647 | for (i = 0; i < IOAT_TEST_SIZE; i++) | |
648 | src[i] = (u8)i; | |
649 | ||
650 | /* Start copy, using first DMA channel */ | |
651 | dma_chan = container_of(device->common.channels.next, | |
652 | struct dma_chan, | |
653 | device_node); | |
654 | if (ioat_dma_alloc_chan_resources(dma_chan) < 1) { | |
655 | err = -ENODEV; | |
656 | goto out; | |
657 | } | |
658 | ||
659 | cookie = ioat_dma_memcpy_buf_to_buf(dma_chan, dest, src, IOAT_TEST_SIZE); | |
660 | ioat_dma_memcpy_issue_pending(dma_chan); | |
661 | msleep(1); | |
662 | ||
663 | if (ioat_dma_is_complete(dma_chan, cookie, NULL, NULL) != DMA_SUCCESS) { | |
664 | printk(KERN_ERR "ioatdma: Self-test copy timed out, disabling\n"); | |
665 | err = -ENODEV; | |
666 | goto free_resources; | |
667 | } | |
668 | if (memcmp(src, dest, IOAT_TEST_SIZE)) { | |
669 | printk(KERN_ERR "ioatdma: Self-test copy failed compare, disabling\n"); | |
670 | err = -ENODEV; | |
671 | goto free_resources; | |
672 | } | |
673 | ||
674 | free_resources: | |
675 | ioat_dma_free_chan_resources(dma_chan); | |
676 | out: | |
677 | kfree(src); | |
678 | kfree(dest); | |
679 | return err; | |
680 | } | |
681 | ||
682 | static int __devinit ioat_probe(struct pci_dev *pdev, | |
683 | const struct pci_device_id *ent) | |
684 | { | |
685 | int err; | |
686 | unsigned long mmio_start, mmio_len; | |
687 | void *reg_base; | |
688 | struct ioat_device *device; | |
689 | ||
690 | err = pci_enable_device(pdev); | |
691 | if (err) | |
692 | goto err_enable_device; | |
693 | ||
694 | err = pci_set_dma_mask(pdev, DMA_64BIT_MASK); | |
695 | if (err) | |
696 | err = pci_set_dma_mask(pdev, DMA_32BIT_MASK); | |
697 | if (err) | |
698 | goto err_set_dma_mask; | |
699 | ||
700 | err = pci_request_regions(pdev, ioat_pci_drv.name); | |
701 | if (err) | |
702 | goto err_request_regions; | |
703 | ||
704 | mmio_start = pci_resource_start(pdev, 0); | |
705 | mmio_len = pci_resource_len(pdev, 0); | |
706 | ||
707 | reg_base = ioremap(mmio_start, mmio_len); | |
708 | if (!reg_base) { | |
709 | err = -ENOMEM; | |
710 | goto err_ioremap; | |
711 | } | |
712 | ||
713 | device = kzalloc(sizeof(*device), GFP_KERNEL); | |
714 | if (!device) { | |
715 | err = -ENOMEM; | |
716 | goto err_kzalloc; | |
717 | } | |
718 | ||
719 | /* DMA coherent memory pool for DMA descriptor allocations */ | |
720 | device->dma_pool = pci_pool_create("dma_desc_pool", pdev, | |
721 | sizeof(struct ioat_dma_descriptor), 64, 0); | |
722 | if (!device->dma_pool) { | |
723 | err = -ENOMEM; | |
724 | goto err_dma_pool; | |
725 | } | |
726 | ||
727 | device->completion_pool = pci_pool_create("completion_pool", pdev, sizeof(u64), SMP_CACHE_BYTES, SMP_CACHE_BYTES); | |
728 | if (!device->completion_pool) { | |
729 | err = -ENOMEM; | |
730 | goto err_completion_pool; | |
731 | } | |
732 | ||
733 | device->pdev = pdev; | |
734 | pci_set_drvdata(pdev, device); | |
735 | #ifdef CONFIG_PCI_MSI | |
736 | if (pci_enable_msi(pdev) == 0) { | |
737 | device->msi = 1; | |
738 | } else { | |
739 | device->msi = 0; | |
740 | } | |
741 | #endif | |
dace1453 | 742 | err = request_irq(pdev->irq, &ioat_do_interrupt, IRQF_SHARED, "ioat", |
0bbd5f4e CL |
743 | device); |
744 | if (err) | |
745 | goto err_irq; | |
746 | ||
747 | device->reg_base = reg_base; | |
748 | ||
749 | ioatdma_write8(device, IOAT_INTRCTRL_OFFSET, IOAT_INTRCTRL_MASTER_INT_EN); | |
750 | pci_set_master(pdev); | |
751 | ||
752 | INIT_LIST_HEAD(&device->common.channels); | |
753 | enumerate_dma_channels(device); | |
754 | ||
755 | device->common.device_alloc_chan_resources = ioat_dma_alloc_chan_resources; | |
756 | device->common.device_free_chan_resources = ioat_dma_free_chan_resources; | |
757 | device->common.device_memcpy_buf_to_buf = ioat_dma_memcpy_buf_to_buf; | |
758 | device->common.device_memcpy_buf_to_pg = ioat_dma_memcpy_buf_to_pg; | |
759 | device->common.device_memcpy_pg_to_pg = ioat_dma_memcpy_pg_to_pg; | |
760 | device->common.device_memcpy_complete = ioat_dma_is_complete; | |
761 | device->common.device_memcpy_issue_pending = ioat_dma_memcpy_issue_pending; | |
762 | printk(KERN_INFO "Intel(R) I/OAT DMA Engine found, %d channels\n", | |
763 | device->common.chancnt); | |
764 | ||
765 | err = ioat_self_test(device); | |
766 | if (err) | |
767 | goto err_self_test; | |
768 | ||
769 | dma_async_device_register(&device->common); | |
770 | ||
771 | return 0; | |
772 | ||
773 | err_self_test: | |
774 | err_irq: | |
775 | pci_pool_destroy(device->completion_pool); | |
776 | err_completion_pool: | |
777 | pci_pool_destroy(device->dma_pool); | |
778 | err_dma_pool: | |
779 | kfree(device); | |
780 | err_kzalloc: | |
781 | iounmap(reg_base); | |
782 | err_ioremap: | |
783 | pci_release_regions(pdev); | |
784 | err_request_regions: | |
785 | err_set_dma_mask: | |
786 | pci_disable_device(pdev); | |
787 | err_enable_device: | |
788 | return err; | |
789 | } | |
790 | ||
791 | static void __devexit ioat_remove(struct pci_dev *pdev) | |
792 | { | |
793 | struct ioat_device *device; | |
794 | struct dma_chan *chan, *_chan; | |
795 | struct ioat_dma_chan *ioat_chan; | |
796 | ||
797 | device = pci_get_drvdata(pdev); | |
798 | dma_async_device_unregister(&device->common); | |
799 | ||
800 | free_irq(device->pdev->irq, device); | |
801 | #ifdef CONFIG_PCI_MSI | |
802 | if (device->msi) | |
803 | pci_disable_msi(device->pdev); | |
804 | #endif | |
805 | pci_pool_destroy(device->dma_pool); | |
806 | pci_pool_destroy(device->completion_pool); | |
807 | iounmap(device->reg_base); | |
808 | pci_release_regions(pdev); | |
809 | pci_disable_device(pdev); | |
810 | list_for_each_entry_safe(chan, _chan, &device->common.channels, device_node) { | |
811 | ioat_chan = to_ioat_chan(chan); | |
812 | list_del(&chan->device_node); | |
813 | kfree(ioat_chan); | |
814 | } | |
815 | kfree(device); | |
816 | } | |
817 | ||
818 | /* MODULE API */ | |
819 | MODULE_VERSION("1.7"); | |
820 | MODULE_LICENSE("GPL"); | |
821 | MODULE_AUTHOR("Intel Corporation"); | |
822 | ||
823 | static int __init ioat_init_module(void) | |
824 | { | |
825 | /* it's currently unsafe to unload this module */ | |
826 | /* if forced, worst case is that rmmod hangs */ | |
8070b2b1 | 827 | __unsafe(THIS_MODULE); |
0bbd5f4e | 828 | |
8070b2b1 | 829 | pci_module_init(&ioat_pci_drv); |
0bbd5f4e CL |
830 | } |
831 | ||
832 | module_init(ioat_init_module); | |
833 | ||
834 | static void __exit ioat_exit_module(void) | |
835 | { | |
836 | pci_unregister_driver(&ioat_pci_drv); | |
837 | } | |
838 | ||
839 | module_exit(ioat_exit_module); |