1 /* SPDX-License-Identifier: GPL-2.0-or-later */
3 * Copyright (C) 2001 Mike Corrigan & Dave Engebretsen, IBM Corporation
5 * Copyright (C) 2004 Olof Johansson <olof@lixom.net>, IBM Corporation
12 #include <linux/compiler.h>
13 #include <linux/spinlock.h>
14 #include <linux/device.h>
15 #include <linux/dma-mapping.h>
16 #include <linux/bitops.h>
17 #include <asm/machdep.h>
18 #include <asm/types.h>
19 #include <asm/pci-bridge.h>
20 #include <asm/asm-const.h>
22 #define IOMMU_PAGE_SHIFT_4K 12
23 #define IOMMU_PAGE_SIZE_4K (ASM_CONST(1) << IOMMU_PAGE_SHIFT_4K)
24 #define IOMMU_PAGE_MASK_4K (~((1 << IOMMU_PAGE_SHIFT_4K) - 1))
25 #define IOMMU_PAGE_ALIGN_4K(addr) _ALIGN_UP(addr, IOMMU_PAGE_SIZE_4K)
27 #define IOMMU_PAGE_SIZE(tblptr) (ASM_CONST(1) << (tblptr)->it_page_shift)
28 #define IOMMU_PAGE_MASK(tblptr) (~((1 << (tblptr)->it_page_shift) - 1))
29 #define IOMMU_PAGE_ALIGN(addr, tblptr) _ALIGN_UP(addr, IOMMU_PAGE_SIZE(tblptr))
32 extern int iommu_is_off
;
33 extern int iommu_force_on
;
35 struct iommu_table_ops
{
37 * When called with direction==DMA_NONE, it is equal to clear().
38 * uaddr is a linear map address.
40 int (*set
)(struct iommu_table
*tbl
,
41 long index
, long npages
,
43 enum dma_data_direction direction
,
45 #ifdef CONFIG_IOMMU_API
47 * Exchanges existing TCE with new TCE plus direction bits;
48 * returns old TCE and DMA direction mask.
49 * @tce is a physical address.
51 int (*exchange
)(struct iommu_table
*tbl
,
54 enum dma_data_direction
*direction
);
56 int (*exchange_rm
)(struct iommu_table
*tbl
,
59 enum dma_data_direction
*direction
);
61 __be64
*(*useraddrptr
)(struct iommu_table
*tbl
, long index
, bool alloc
);
63 void (*clear
)(struct iommu_table
*tbl
,
64 long index
, long npages
);
65 /* get() returns a physical address */
66 unsigned long (*get
)(struct iommu_table
*tbl
, long index
);
67 void (*flush
)(struct iommu_table
*tbl
);
68 void (*free
)(struct iommu_table
*tbl
);
71 /* These are used by VIO */
72 extern struct iommu_table_ops iommu_table_lpar_multi_ops
;
73 extern struct iommu_table_ops iommu_table_pseries_ops
;
76 * IOMAP_MAX_ORDER defines the largest contiguous block
77 * of dma space we can get. IOMAP_MAX_ORDER = 13
78 * allows up to 2**12 pages (4096 * 4096) = 16 MB
80 #define IOMAP_MAX_ORDER 13
82 #define IOMMU_POOL_HASHBITS 2
83 #define IOMMU_NR_POOLS (1 << IOMMU_POOL_HASHBITS)
90 } ____cacheline_aligned_in_smp
;
93 unsigned long it_busno
; /* Bus number this table belongs to */
94 unsigned long it_size
; /* Size of iommu table in entries */
95 unsigned long it_indirect_levels
;
96 unsigned long it_level_size
;
97 unsigned long it_allocated_size
;
98 unsigned long it_offset
; /* Offset into global table */
99 unsigned long it_base
; /* mapped address of tce table */
100 unsigned long it_index
; /* which iommu table this is */
101 unsigned long it_type
; /* type: PCI or Virtual Bus */
102 unsigned long it_blocksize
; /* Entries in each block (cacheline) */
103 unsigned long poolsize
;
104 unsigned long nr_pools
;
105 struct iommu_pool large_pool
;
106 struct iommu_pool pools
[IOMMU_NR_POOLS
];
107 unsigned long *it_map
; /* A simple allocation bitmap for now */
108 unsigned long it_page_shift
;/* table iommu page size */
109 struct list_head it_group_list
;/* List of iommu_table_group_link */
110 __be64
*it_userspace
; /* userspace view of the table */
111 struct iommu_table_ops
*it_ops
;
116 #define IOMMU_TABLE_USERSPACE_ENTRY_RO(tbl, entry) \
117 ((tbl)->it_ops->useraddrptr((tbl), (entry), false))
118 #define IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry) \
119 ((tbl)->it_ops->useraddrptr((tbl), (entry), true))
121 /* Pure 2^n version of get_order */
122 static inline __attribute_const__
123 int get_iommu_order(unsigned long size
, struct iommu_table
*tbl
)
125 return __ilog2((size
- 1) >> tbl
->it_page_shift
) + 1;
133 static inline void set_iommu_table_base(struct device
*dev
,
134 struct iommu_table
*base
)
136 dev
->archdata
.iommu_table_base
= base
;
139 static inline void *get_iommu_table_base(struct device
*dev
)
141 return dev
->archdata
.iommu_table_base
;
144 extern int dma_iommu_dma_supported(struct device
*dev
, u64 mask
);
146 extern struct iommu_table
*iommu_tce_table_get(struct iommu_table
*tbl
);
147 extern int iommu_tce_table_put(struct iommu_table
*tbl
);
149 /* Initializes an iommu_table based in values set in the passed-in
152 extern struct iommu_table
*iommu_init_table(struct iommu_table
* tbl
,
154 #define IOMMU_TABLE_GROUP_MAX_TABLES 2
156 struct iommu_table_group
;
158 struct iommu_table_group_ops
{
159 unsigned long (*get_table_size
)(
163 long (*create_table
)(struct iommu_table_group
*table_group
,
168 struct iommu_table
**ptbl
);
169 long (*set_window
)(struct iommu_table_group
*table_group
,
171 struct iommu_table
*tblnew
);
172 long (*unset_window
)(struct iommu_table_group
*table_group
,
174 /* Switch ownership from platform code to external user (e.g. VFIO) */
175 void (*take_ownership
)(struct iommu_table_group
*table_group
);
176 /* Switch ownership from external user (e.g. VFIO) back to core */
177 void (*release_ownership
)(struct iommu_table_group
*table_group
);
180 struct iommu_table_group_link
{
181 struct list_head next
;
183 struct iommu_table_group
*table_group
;
186 struct iommu_table_group
{
187 /* IOMMU properties */
190 __u64 pgsizes
; /* Bitmap of supported page sizes */
191 __u32 max_dynamic_windows_supported
;
194 struct iommu_group
*group
;
195 struct iommu_table
*tables
[IOMMU_TABLE_GROUP_MAX_TABLES
];
196 struct iommu_table_group_ops
*ops
;
199 #ifdef CONFIG_IOMMU_API
201 extern void iommu_register_group(struct iommu_table_group
*table_group
,
202 int pci_domain_number
, unsigned long pe_num
);
203 extern int iommu_add_device(struct iommu_table_group
*table_group
,
205 extern void iommu_del_device(struct device
*dev
);
206 extern long iommu_tce_xchg(struct mm_struct
*mm
, struct iommu_table
*tbl
,
207 unsigned long entry
, unsigned long *hpa
,
208 enum dma_data_direction
*direction
);
210 static inline void iommu_register_group(struct iommu_table_group
*table_group
,
211 int pci_domain_number
,
212 unsigned long pe_num
)
216 static inline int iommu_add_device(struct iommu_table_group
*table_group
,
222 static inline void iommu_del_device(struct device
*dev
)
225 #endif /* !CONFIG_IOMMU_API */
227 u64
dma_iommu_get_required_mask(struct device
*dev
);
230 static inline void *get_iommu_table_base(struct device
*dev
)
235 static inline int dma_iommu_dma_supported(struct device
*dev
, u64 mask
)
240 #endif /* CONFIG_PPC64 */
242 extern int ppc_iommu_map_sg(struct device
*dev
, struct iommu_table
*tbl
,
243 struct scatterlist
*sglist
, int nelems
,
245 enum dma_data_direction direction
,
246 unsigned long attrs
);
247 extern void ppc_iommu_unmap_sg(struct iommu_table
*tbl
,
248 struct scatterlist
*sglist
,
250 enum dma_data_direction direction
,
251 unsigned long attrs
);
253 extern void *iommu_alloc_coherent(struct device
*dev
, struct iommu_table
*tbl
,
254 size_t size
, dma_addr_t
*dma_handle
,
255 unsigned long mask
, gfp_t flag
, int node
);
256 extern void iommu_free_coherent(struct iommu_table
*tbl
, size_t size
,
257 void *vaddr
, dma_addr_t dma_handle
);
258 extern dma_addr_t
iommu_map_page(struct device
*dev
, struct iommu_table
*tbl
,
259 struct page
*page
, unsigned long offset
,
260 size_t size
, unsigned long mask
,
261 enum dma_data_direction direction
,
262 unsigned long attrs
);
263 extern void iommu_unmap_page(struct iommu_table
*tbl
, dma_addr_t dma_handle
,
264 size_t size
, enum dma_data_direction direction
,
265 unsigned long attrs
);
267 extern void iommu_init_early_pSeries(void);
268 extern void iommu_init_early_dart(struct pci_controller_ops
*controller_ops
);
269 extern void iommu_init_early_pasemi(void);
271 #if defined(CONFIG_PPC64) && defined(CONFIG_PM)
272 static inline void iommu_save(void)
274 if (ppc_md
.iommu_save
)
278 static inline void iommu_restore(void)
280 if (ppc_md
.iommu_restore
)
281 ppc_md
.iommu_restore();
285 /* The API to support IOMMU operations for VFIO */
286 extern int iommu_tce_check_ioba(unsigned long page_shift
,
287 unsigned long offset
, unsigned long size
,
288 unsigned long ioba
, unsigned long npages
);
289 extern int iommu_tce_check_gpa(unsigned long page_shift
,
292 #define iommu_tce_clear_param_check(tbl, ioba, tce_value, npages) \
293 (iommu_tce_check_ioba((tbl)->it_page_shift, \
294 (tbl)->it_offset, (tbl)->it_size, \
295 (ioba), (npages)) || (tce_value))
296 #define iommu_tce_put_param_check(tbl, ioba, gpa) \
297 (iommu_tce_check_ioba((tbl)->it_page_shift, \
298 (tbl)->it_offset, (tbl)->it_size, \
300 iommu_tce_check_gpa((tbl)->it_page_shift, (gpa)))
302 extern void iommu_flush_tce(struct iommu_table
*tbl
);
303 extern int iommu_take_ownership(struct iommu_table
*tbl
);
304 extern void iommu_release_ownership(struct iommu_table
*tbl
);
306 extern enum dma_data_direction
iommu_tce_direction(unsigned long tce
);
307 extern unsigned long iommu_direction_to_tce_perm(enum dma_data_direction dir
);
309 #ifdef CONFIG_PPC_CELL_NATIVE
310 extern bool iommu_fixed_is_weak
;
312 #define iommu_fixed_is_weak false
315 extern const struct dma_map_ops dma_iommu_ops
;
317 static inline unsigned long device_to_mask(struct device
*dev
)
319 if (dev
->dma_mask
&& *dev
->dma_mask
)
320 return *dev
->dma_mask
;
321 /* Assume devices without mask can take 32 bit addresses */
325 #endif /* __KERNEL__ */
326 #endif /* _ASM_IOMMU_H */