2 * Copyright IBM Corp. 2012
5 * Jan Glauber <jang@linux.vnet.ibm.com>
8 #include <linux/kernel.h>
9 #include <linux/slab.h>
10 #include <linux/export.h>
11 #include <linux/iommu-helper.h>
12 #include <linux/dma-mapping.h>
13 #include <linux/vmalloc.h>
14 #include <linux/pci.h>
15 #include <asm/pci_dma.h>
17 static struct kmem_cache
*dma_region_table_cache
;
18 static struct kmem_cache
*dma_page_table_cache
;
19 static int s390_iommu_strict
;
21 static int zpci_refresh_global(struct zpci_dev
*zdev
)
23 return zpci_refresh_trans((u64
) zdev
->fh
<< 32, zdev
->start_dma
,
24 zdev
->iommu_pages
* PAGE_SIZE
);
27 unsigned long *dma_alloc_cpu_table(void)
29 unsigned long *table
, *entry
;
31 table
= kmem_cache_alloc(dma_region_table_cache
, GFP_ATOMIC
);
35 for (entry
= table
; entry
< table
+ ZPCI_TABLE_ENTRIES
; entry
++)
36 *entry
= ZPCI_TABLE_INVALID
| ZPCI_TABLE_PROTECTED
;
40 static void dma_free_cpu_table(void *table
)
42 kmem_cache_free(dma_region_table_cache
, table
);
45 static unsigned long *dma_alloc_page_table(void)
47 unsigned long *table
, *entry
;
49 table
= kmem_cache_alloc(dma_page_table_cache
, GFP_ATOMIC
);
53 for (entry
= table
; entry
< table
+ ZPCI_PT_ENTRIES
; entry
++)
54 *entry
= ZPCI_PTE_INVALID
| ZPCI_TABLE_PROTECTED
;
58 static void dma_free_page_table(void *table
)
60 kmem_cache_free(dma_page_table_cache
, table
);
63 static unsigned long *dma_get_seg_table_origin(unsigned long *entry
)
67 if (reg_entry_isvalid(*entry
))
68 sto
= get_rt_sto(*entry
);
70 sto
= dma_alloc_cpu_table();
74 set_rt_sto(entry
, sto
);
75 validate_rt_entry(entry
);
76 entry_clr_protected(entry
);
81 static unsigned long *dma_get_page_table_origin(unsigned long *entry
)
85 if (reg_entry_isvalid(*entry
))
86 pto
= get_st_pto(*entry
);
88 pto
= dma_alloc_page_table();
91 set_st_pto(entry
, pto
);
92 validate_st_entry(entry
);
93 entry_clr_protected(entry
);
98 static unsigned long *dma_walk_cpu_trans(unsigned long *rto
, dma_addr_t dma_addr
)
100 unsigned long *sto
, *pto
;
101 unsigned int rtx
, sx
, px
;
103 rtx
= calc_rtx(dma_addr
);
104 sto
= dma_get_seg_table_origin(&rto
[rtx
]);
108 sx
= calc_sx(dma_addr
);
109 pto
= dma_get_page_table_origin(&sto
[sx
]);
113 px
= calc_px(dma_addr
);
117 void dma_update_cpu_trans(unsigned long *dma_table
, void *page_addr
,
118 dma_addr_t dma_addr
, int flags
)
120 unsigned long *entry
;
122 entry
= dma_walk_cpu_trans(dma_table
, dma_addr
);
128 if (flags
& ZPCI_PTE_INVALID
) {
129 invalidate_pt_entry(entry
);
132 set_pt_pfaa(entry
, page_addr
);
133 validate_pt_entry(entry
);
136 if (flags
& ZPCI_TABLE_PROTECTED
)
137 entry_set_protected(entry
);
139 entry_clr_protected(entry
);
142 static int dma_update_trans(struct zpci_dev
*zdev
, unsigned long pa
,
143 dma_addr_t dma_addr
, size_t size
, int flags
)
145 unsigned int nr_pages
= PAGE_ALIGN(size
) >> PAGE_SHIFT
;
146 u8
*page_addr
= (u8
*) (pa
& PAGE_MASK
);
147 dma_addr_t start_dma_addr
= dma_addr
;
148 unsigned long irq_flags
;
154 spin_lock_irqsave(&zdev
->dma_table_lock
, irq_flags
);
155 if (!zdev
->dma_table
)
158 for (i
= 0; i
< nr_pages
; i
++) {
159 dma_update_cpu_trans(zdev
->dma_table
, page_addr
, dma_addr
,
161 page_addr
+= PAGE_SIZE
;
162 dma_addr
+= PAGE_SIZE
;
166 * With zdev->tlb_refresh == 0, rpcit is not required to establish new
167 * translations when previously invalid translation-table entries are
168 * validated. With lazy unmap, it also is skipped for previously valid
169 * entries, but a global rpcit is then required before any address can
170 * be re-used, i.e. after each iommu bitmap wrap-around.
172 if (!zdev
->tlb_refresh
&&
173 (!s390_iommu_strict
||
174 ((flags
& ZPCI_PTE_VALID_MASK
) == ZPCI_PTE_VALID
)))
177 rc
= zpci_refresh_trans((u64
) zdev
->fh
<< 32, start_dma_addr
,
178 nr_pages
* PAGE_SIZE
);
181 spin_unlock_irqrestore(&zdev
->dma_table_lock
, irq_flags
);
185 void dma_free_seg_table(unsigned long entry
)
187 unsigned long *sto
= get_rt_sto(entry
);
190 for (sx
= 0; sx
< ZPCI_TABLE_ENTRIES
; sx
++)
191 if (reg_entry_isvalid(sto
[sx
]))
192 dma_free_page_table(get_st_pto(sto
[sx
]));
194 dma_free_cpu_table(sto
);
197 void dma_cleanup_tables(unsigned long *table
)
204 for (rtx
= 0; rtx
< ZPCI_TABLE_ENTRIES
; rtx
++)
205 if (reg_entry_isvalid(table
[rtx
]))
206 dma_free_seg_table(table
[rtx
]);
208 dma_free_cpu_table(table
);
211 static unsigned long __dma_alloc_iommu(struct zpci_dev
*zdev
,
212 unsigned long start
, int size
)
214 unsigned long boundary_size
;
216 boundary_size
= ALIGN(dma_get_seg_boundary(&zdev
->pdev
->dev
) + 1,
217 PAGE_SIZE
) >> PAGE_SHIFT
;
218 return iommu_area_alloc(zdev
->iommu_bitmap
, zdev
->iommu_pages
,
219 start
, size
, 0, boundary_size
, 0);
222 static unsigned long dma_alloc_iommu(struct zpci_dev
*zdev
, int size
)
224 unsigned long offset
, flags
;
227 spin_lock_irqsave(&zdev
->iommu_bitmap_lock
, flags
);
228 offset
= __dma_alloc_iommu(zdev
, zdev
->next_bit
, size
);
231 offset
= __dma_alloc_iommu(zdev
, 0, size
);
236 zdev
->next_bit
= offset
+ size
;
237 if (!zdev
->tlb_refresh
&& !s390_iommu_strict
&& wrap
)
238 /* global flush after wrap-around with lazy unmap */
239 zpci_refresh_global(zdev
);
241 spin_unlock_irqrestore(&zdev
->iommu_bitmap_lock
, flags
);
245 static void dma_free_iommu(struct zpci_dev
*zdev
, unsigned long offset
, int size
)
249 spin_lock_irqsave(&zdev
->iommu_bitmap_lock
, flags
);
250 if (!zdev
->iommu_bitmap
)
252 bitmap_clear(zdev
->iommu_bitmap
, offset
, size
);
254 * Lazy flush for unmap: need to move next_bit to avoid address re-use
257 if (!s390_iommu_strict
&& offset
>= zdev
->next_bit
)
258 zdev
->next_bit
= offset
+ size
;
260 spin_unlock_irqrestore(&zdev
->iommu_bitmap_lock
, flags
);
263 static dma_addr_t
s390_dma_map_pages(struct device
*dev
, struct page
*page
,
264 unsigned long offset
, size_t size
,
265 enum dma_data_direction direction
,
266 struct dma_attrs
*attrs
)
268 struct zpci_dev
*zdev
= to_zpci(to_pci_dev(dev
));
269 unsigned long nr_pages
, iommu_page_index
;
270 unsigned long pa
= page_to_phys(page
) + offset
;
271 int flags
= ZPCI_PTE_VALID
;
274 /* This rounds up number of pages based on size and offset */
275 nr_pages
= iommu_num_pages(pa
, size
, PAGE_SIZE
);
276 iommu_page_index
= dma_alloc_iommu(zdev
, nr_pages
);
277 if (iommu_page_index
== -1)
280 /* Use rounded up size */
281 size
= nr_pages
* PAGE_SIZE
;
283 dma_addr
= zdev
->start_dma
+ iommu_page_index
* PAGE_SIZE
;
284 if (dma_addr
+ size
> zdev
->end_dma
)
287 if (direction
== DMA_NONE
|| direction
== DMA_TO_DEVICE
)
288 flags
|= ZPCI_TABLE_PROTECTED
;
290 if (!dma_update_trans(zdev
, pa
, dma_addr
, size
, flags
)) {
291 atomic64_add(nr_pages
, &zdev
->mapped_pages
);
292 return dma_addr
+ (offset
& ~PAGE_MASK
);
296 dma_free_iommu(zdev
, iommu_page_index
, nr_pages
);
298 zpci_err("map error:\n");
299 zpci_err_hex(&pa
, sizeof(pa
));
300 return DMA_ERROR_CODE
;
303 static void s390_dma_unmap_pages(struct device
*dev
, dma_addr_t dma_addr
,
304 size_t size
, enum dma_data_direction direction
,
305 struct dma_attrs
*attrs
)
307 struct zpci_dev
*zdev
= to_zpci(to_pci_dev(dev
));
308 unsigned long iommu_page_index
;
311 npages
= iommu_num_pages(dma_addr
, size
, PAGE_SIZE
);
312 dma_addr
= dma_addr
& PAGE_MASK
;
313 if (dma_update_trans(zdev
, 0, dma_addr
, npages
* PAGE_SIZE
,
314 ZPCI_TABLE_PROTECTED
| ZPCI_PTE_INVALID
)) {
315 zpci_err("unmap error:\n");
316 zpci_err_hex(&dma_addr
, sizeof(dma_addr
));
319 atomic64_add(npages
, &zdev
->unmapped_pages
);
320 iommu_page_index
= (dma_addr
- zdev
->start_dma
) >> PAGE_SHIFT
;
321 dma_free_iommu(zdev
, iommu_page_index
, npages
);
324 static void *s390_dma_alloc(struct device
*dev
, size_t size
,
325 dma_addr_t
*dma_handle
, gfp_t flag
,
326 struct dma_attrs
*attrs
)
328 struct zpci_dev
*zdev
= to_zpci(to_pci_dev(dev
));
333 size
= PAGE_ALIGN(size
);
334 page
= alloc_pages(flag
, get_order(size
));
338 pa
= page_to_phys(page
);
339 memset((void *) pa
, 0, size
);
341 map
= s390_dma_map_pages(dev
, page
, pa
% PAGE_SIZE
,
342 size
, DMA_BIDIRECTIONAL
, NULL
);
343 if (dma_mapping_error(dev
, map
)) {
344 free_pages(pa
, get_order(size
));
348 atomic64_add(size
/ PAGE_SIZE
, &zdev
->allocated_pages
);
354 static void s390_dma_free(struct device
*dev
, size_t size
,
355 void *pa
, dma_addr_t dma_handle
,
356 struct dma_attrs
*attrs
)
358 struct zpci_dev
*zdev
= to_zpci(to_pci_dev(dev
));
360 size
= PAGE_ALIGN(size
);
361 atomic64_sub(size
/ PAGE_SIZE
, &zdev
->allocated_pages
);
362 s390_dma_unmap_pages(dev
, dma_handle
, size
, DMA_BIDIRECTIONAL
, NULL
);
363 free_pages((unsigned long) pa
, get_order(size
));
366 static int s390_dma_map_sg(struct device
*dev
, struct scatterlist
*sg
,
367 int nr_elements
, enum dma_data_direction dir
,
368 struct dma_attrs
*attrs
)
370 int mapped_elements
= 0;
371 struct scatterlist
*s
;
374 for_each_sg(sg
, s
, nr_elements
, i
) {
375 struct page
*page
= sg_page(s
);
376 s
->dma_address
= s390_dma_map_pages(dev
, page
, s
->offset
,
377 s
->length
, dir
, NULL
);
378 if (!dma_mapping_error(dev
, s
->dma_address
)) {
379 s
->dma_length
= s
->length
;
385 return mapped_elements
;
388 for_each_sg(sg
, s
, mapped_elements
, i
) {
390 s390_dma_unmap_pages(dev
, s
->dma_address
, s
->dma_length
,
399 static void s390_dma_unmap_sg(struct device
*dev
, struct scatterlist
*sg
,
400 int nr_elements
, enum dma_data_direction dir
,
401 struct dma_attrs
*attrs
)
403 struct scatterlist
*s
;
406 for_each_sg(sg
, s
, nr_elements
, i
) {
407 s390_dma_unmap_pages(dev
, s
->dma_address
, s
->dma_length
, dir
, NULL
);
413 int zpci_dma_init_device(struct zpci_dev
*zdev
)
418 * At this point, if the device is part of an IOMMU domain, this would
419 * be a strong hint towards a bug in the IOMMU API (common) code and/or
420 * simultaneous access via IOMMU and DMA API. So let's issue a warning.
422 WARN_ON(zdev
->s390_domain
);
424 spin_lock_init(&zdev
->iommu_bitmap_lock
);
425 spin_lock_init(&zdev
->dma_table_lock
);
427 zdev
->dma_table
= dma_alloc_cpu_table();
428 if (!zdev
->dma_table
) {
433 zdev
->iommu_size
= (unsigned long) high_memory
- PAGE_OFFSET
;
434 zdev
->iommu_pages
= zdev
->iommu_size
>> PAGE_SHIFT
;
435 zdev
->iommu_bitmap
= vzalloc(zdev
->iommu_pages
/ 8);
436 if (!zdev
->iommu_bitmap
) {
441 rc
= zpci_register_ioat(zdev
,
443 zdev
->start_dma
+ PAGE_OFFSET
,
444 zdev
->start_dma
+ zdev
->iommu_size
- 1,
445 (u64
) zdev
->dma_table
);
451 dma_free_cpu_table(zdev
->dma_table
);
456 void zpci_dma_exit_device(struct zpci_dev
*zdev
)
459 * At this point, if the device is part of an IOMMU domain, this would
460 * be a strong hint towards a bug in the IOMMU API (common) code and/or
461 * simultaneous access via IOMMU and DMA API. So let's issue a warning.
463 WARN_ON(zdev
->s390_domain
);
465 zpci_unregister_ioat(zdev
, 0);
466 dma_cleanup_tables(zdev
->dma_table
);
467 zdev
->dma_table
= NULL
;
468 vfree(zdev
->iommu_bitmap
);
469 zdev
->iommu_bitmap
= NULL
;
473 static int __init
dma_alloc_cpu_table_caches(void)
475 dma_region_table_cache
= kmem_cache_create("PCI_DMA_region_tables",
476 ZPCI_TABLE_SIZE
, ZPCI_TABLE_ALIGN
,
478 if (!dma_region_table_cache
)
481 dma_page_table_cache
= kmem_cache_create("PCI_DMA_page_tables",
482 ZPCI_PT_SIZE
, ZPCI_PT_ALIGN
,
484 if (!dma_page_table_cache
) {
485 kmem_cache_destroy(dma_region_table_cache
);
491 int __init
zpci_dma_init(void)
493 return dma_alloc_cpu_table_caches();
496 void zpci_dma_exit(void)
498 kmem_cache_destroy(dma_page_table_cache
);
499 kmem_cache_destroy(dma_region_table_cache
);
502 #define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16)
504 static int __init
dma_debug_do_init(void)
506 dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES
);
509 fs_initcall(dma_debug_do_init
);
511 struct dma_map_ops s390_dma_ops
= {
512 .alloc
= s390_dma_alloc
,
513 .free
= s390_dma_free
,
514 .map_sg
= s390_dma_map_sg
,
515 .unmap_sg
= s390_dma_unmap_sg
,
516 .map_page
= s390_dma_map_pages
,
517 .unmap_page
= s390_dma_unmap_pages
,
518 /* if we support direct DMA this must be conditional */
520 /* dma_supported is unconditionally true without a callback */
522 EXPORT_SYMBOL_GPL(s390_dma_ops
);
524 static int __init
s390_iommu_setup(char *str
)
526 if (!strncmp(str
, "strict", 6))
527 s390_iommu_strict
= 1;
531 __setup("s390_iommu=", s390_iommu_setup
);