1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright (C) 2006 Benjamin Herrenschmidt, IBM Corporation
5 * Provide default implementations of the DMA mapping callbacks for
6 * busses using the iommu infrastructure
9 #include <linux/dma-direct.h>
10 #include <linux/pci.h>
11 #include <asm/iommu.h>
14 * Generic iommu implementation
18 * The coherent mask may be smaller than the real mask, check if we can
19 * really use a direct window.
21 static inline bool dma_iommu_alloc_bypass(struct device
*dev
)
23 return dev
->archdata
.iommu_bypass
&& !iommu_fixed_is_weak
&&
24 dma_direct_supported(dev
, dev
->coherent_dma_mask
);
27 static inline bool dma_iommu_map_bypass(struct device
*dev
,
30 return dev
->archdata
.iommu_bypass
&&
31 (!iommu_fixed_is_weak
|| (attrs
& DMA_ATTR_WEAK_ORDERING
));
34 /* Allocates a contiguous real buffer and creates mappings over it.
35 * Returns the virtual address of the buffer and sets dma_handle
36 * to the dma address (mapping) of the first page.
38 static void *dma_iommu_alloc_coherent(struct device
*dev
, size_t size
,
39 dma_addr_t
*dma_handle
, gfp_t flag
,
42 if (dma_iommu_alloc_bypass(dev
))
43 return dma_direct_alloc(dev
, size
, dma_handle
, flag
, attrs
);
44 return iommu_alloc_coherent(dev
, get_iommu_table_base(dev
), size
,
45 dma_handle
, dev
->coherent_dma_mask
, flag
,
49 static void dma_iommu_free_coherent(struct device
*dev
, size_t size
,
50 void *vaddr
, dma_addr_t dma_handle
,
53 if (dma_iommu_alloc_bypass(dev
))
54 dma_direct_free(dev
, size
, vaddr
, dma_handle
, attrs
);
56 iommu_free_coherent(get_iommu_table_base(dev
), size
, vaddr
,
60 /* Creates TCEs for a user provided buffer. The user buffer must be
61 * contiguous real kernel storage (not vmalloc). The address passed here
62 * comprises a page address and offset into that page. The dma_addr_t
63 * returned will point to the same byte within the page as was passed in.
65 static dma_addr_t
dma_iommu_map_page(struct device
*dev
, struct page
*page
,
66 unsigned long offset
, size_t size
,
67 enum dma_data_direction direction
,
70 if (dma_iommu_map_bypass(dev
, attrs
))
71 return dma_direct_map_page(dev
, page
, offset
, size
, direction
,
73 return iommu_map_page(dev
, get_iommu_table_base(dev
), page
, offset
,
74 size
, device_to_mask(dev
), direction
, attrs
);
78 static void dma_iommu_unmap_page(struct device
*dev
, dma_addr_t dma_handle
,
79 size_t size
, enum dma_data_direction direction
,
82 if (!dma_iommu_map_bypass(dev
, attrs
))
83 iommu_unmap_page(get_iommu_table_base(dev
), dma_handle
, size
,
88 static int dma_iommu_map_sg(struct device
*dev
, struct scatterlist
*sglist
,
89 int nelems
, enum dma_data_direction direction
,
92 if (dma_iommu_map_bypass(dev
, attrs
))
93 return dma_direct_map_sg(dev
, sglist
, nelems
, direction
, attrs
);
94 return ppc_iommu_map_sg(dev
, get_iommu_table_base(dev
), sglist
, nelems
,
95 device_to_mask(dev
), direction
, attrs
);
98 static void dma_iommu_unmap_sg(struct device
*dev
, struct scatterlist
*sglist
,
99 int nelems
, enum dma_data_direction direction
,
102 if (!dma_iommu_map_bypass(dev
, attrs
))
103 ppc_iommu_unmap_sg(get_iommu_table_base(dev
), sglist
, nelems
,
107 static bool dma_iommu_bypass_supported(struct device
*dev
, u64 mask
)
109 struct pci_dev
*pdev
= to_pci_dev(dev
);
110 struct pci_controller
*phb
= pci_bus_to_host(pdev
->bus
);
112 return phb
->controller_ops
.iommu_bypass_supported
&&
113 phb
->controller_ops
.iommu_bypass_supported(pdev
, mask
);
116 /* We support DMA to/from any memory page via the iommu */
117 int dma_iommu_dma_supported(struct device
*dev
, u64 mask
)
119 struct iommu_table
*tbl
= get_iommu_table_base(dev
);
122 dev_info(dev
, "Warning: IOMMU dma not supported: mask 0x%08llx"
123 ", table unavailable\n", mask
);
127 if (dev_is_pci(dev
) && dma_iommu_bypass_supported(dev
, mask
)) {
128 dev
->archdata
.iommu_bypass
= true;
129 dev_dbg(dev
, "iommu: 64-bit OK, using fixed ops\n");
133 if (tbl
->it_offset
> (mask
>> tbl
->it_page_shift
)) {
134 dev_info(dev
, "Warning: IOMMU offset too big for device mask\n");
135 dev_info(dev
, "mask: 0x%08llx, table offset: 0x%08lx\n",
136 mask
, tbl
->it_offset
<< tbl
->it_page_shift
);
140 dev_dbg(dev
, "iommu: not 64-bit, using default ops\n");
141 dev
->archdata
.iommu_bypass
= false;
145 u64
dma_iommu_get_required_mask(struct device
*dev
)
147 struct iommu_table
*tbl
= get_iommu_table_base(dev
);
153 if (dev_is_pci(dev
)) {
154 u64 bypass_mask
= dma_direct_get_required_mask(dev
);
156 if (dma_iommu_bypass_supported(dev
, bypass_mask
))
160 mask
= 1ULL < (fls_long(tbl
->it_offset
+ tbl
->it_size
) - 1);
166 const struct dma_map_ops dma_iommu_ops
= {
167 .alloc
= dma_iommu_alloc_coherent
,
168 .free
= dma_iommu_free_coherent
,
169 .map_sg
= dma_iommu_map_sg
,
170 .unmap_sg
= dma_iommu_unmap_sg
,
171 .dma_supported
= dma_iommu_dma_supported
,
172 .map_page
= dma_iommu_map_page
,
173 .unmap_page
= dma_iommu_unmap_page
,
174 .get_required_mask
= dma_iommu_get_required_mask
,