]>
Commit | Line | Data |
---|---|---|
fc2100eb JR |
1 | /* |
2 | * Copyright (C) 2007-2008 Advanced Micro Devices, Inc. | |
3 | * Author: Joerg Roedel <joerg.roedel@amd.com> | |
4 | * | |
5 | * This program is free software; you can redistribute it and/or modify it | |
6 | * under the terms of the GNU General Public License version 2 as published | |
7 | * by the Free Software Foundation. | |
8 | * | |
9 | * This program is distributed in the hope that it will be useful, | |
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
12 | * GNU General Public License for more details. | |
13 | * | |
14 | * You should have received a copy of the GNU General Public License | |
15 | * along with this program; if not, write to the Free Software | |
16 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA | |
17 | */ | |
18 | ||
7d3002cc OBC |
19 | #define pr_fmt(fmt) "%s: " fmt, __func__ |
20 | ||
905d66c1 | 21 | #include <linux/device.h> |
40998188 | 22 | #include <linux/kernel.h> |
fc2100eb JR |
23 | #include <linux/bug.h> |
24 | #include <linux/types.h> | |
60db4027 AM |
25 | #include <linux/module.h> |
26 | #include <linux/slab.h> | |
fc2100eb JR |
27 | #include <linux/errno.h> |
28 | #include <linux/iommu.h> | |
29 | ||
1460432c AW |
30 | static ssize_t show_iommu_group(struct device *dev, |
31 | struct device_attribute *attr, char *buf) | |
32 | { | |
33 | unsigned int groupid; | |
34 | ||
35 | if (iommu_device_group(dev, &groupid)) | |
36 | return 0; | |
37 | ||
38 | return sprintf(buf, "%u", groupid); | |
39 | } | |
40 | static DEVICE_ATTR(iommu_group, S_IRUGO, show_iommu_group, NULL); | |
41 | ||
42 | static int add_iommu_group(struct device *dev, void *data) | |
43 | { | |
44 | unsigned int groupid; | |
45 | ||
46 | if (iommu_device_group(dev, &groupid) == 0) | |
47 | return device_create_file(dev, &dev_attr_iommu_group); | |
48 | ||
49 | return 0; | |
50 | } | |
51 | ||
52 | static int remove_iommu_group(struct device *dev) | |
53 | { | |
54 | unsigned int groupid; | |
55 | ||
56 | if (iommu_device_group(dev, &groupid) == 0) | |
57 | device_remove_file(dev, &dev_attr_iommu_group); | |
58 | ||
59 | return 0; | |
60 | } | |
61 | ||
62 | static int iommu_device_notifier(struct notifier_block *nb, | |
63 | unsigned long action, void *data) | |
64 | { | |
65 | struct device *dev = data; | |
66 | ||
67 | if (action == BUS_NOTIFY_ADD_DEVICE) | |
68 | return add_iommu_group(dev, NULL); | |
69 | else if (action == BUS_NOTIFY_DEL_DEVICE) | |
70 | return remove_iommu_group(dev); | |
71 | ||
72 | return 0; | |
73 | } | |
74 | ||
75 | static struct notifier_block iommu_device_nb = { | |
76 | .notifier_call = iommu_device_notifier, | |
77 | }; | |
78 | ||
ff21776d JR |
79 | static void iommu_bus_init(struct bus_type *bus, struct iommu_ops *ops) |
80 | { | |
1460432c AW |
81 | bus_register_notifier(bus, &iommu_device_nb); |
82 | bus_for_each_dev(bus, NULL, NULL, add_iommu_group); | |
ff21776d | 83 | } |
fc2100eb | 84 | |
ff21776d JR |
85 | /** |
86 | * bus_set_iommu - set iommu-callbacks for the bus | |
87 | * @bus: bus. | |
88 | * @ops: the callbacks provided by the iommu-driver | |
89 | * | |
90 | * This function is called by an iommu driver to set the iommu methods | |
91 | * used for a particular bus. Drivers for devices on that bus can use | |
92 | * the iommu-api after these ops are registered. | |
93 | * This special function is needed because IOMMUs are usually devices on | |
94 | * the bus itself, so the iommu drivers are not initialized when the bus | |
95 | * is set up. With this function the iommu-driver can set the iommu-ops | |
96 | * afterwards. | |
97 | */ | |
98 | int bus_set_iommu(struct bus_type *bus, struct iommu_ops *ops) | |
fc2100eb | 99 | { |
ff21776d JR |
100 | if (bus->iommu_ops != NULL) |
101 | return -EBUSY; | |
fc2100eb | 102 | |
ff21776d JR |
103 | bus->iommu_ops = ops; |
104 | ||
105 | /* Do IOMMU specific setup for this bus-type */ | |
106 | iommu_bus_init(bus, ops); | |
fc2100eb | 107 | |
ff21776d | 108 | return 0; |
fc2100eb | 109 | } |
ff21776d | 110 | EXPORT_SYMBOL_GPL(bus_set_iommu); |
fc2100eb | 111 | |
a1b60c1c | 112 | bool iommu_present(struct bus_type *bus) |
fc2100eb | 113 | { |
94441c3b | 114 | return bus->iommu_ops != NULL; |
fc2100eb | 115 | } |
a1b60c1c | 116 | EXPORT_SYMBOL_GPL(iommu_present); |
fc2100eb | 117 | |
4f3f8d9d OBC |
118 | /** |
119 | * iommu_set_fault_handler() - set a fault handler for an iommu domain | |
120 | * @domain: iommu domain | |
121 | * @handler: fault handler | |
77ca2332 | 122 | * @token: user data, will be passed back to the fault handler |
0ed6d2d2 OBC |
123 | * |
124 | * This function should be used by IOMMU users which want to be notified | |
125 | * whenever an IOMMU fault happens. | |
126 | * | |
127 | * The fault handler itself should return 0 on success, and an appropriate | |
128 | * error code otherwise. | |
4f3f8d9d OBC |
129 | */ |
130 | void iommu_set_fault_handler(struct iommu_domain *domain, | |
77ca2332 OBC |
131 | iommu_fault_handler_t handler, |
132 | void *token) | |
4f3f8d9d OBC |
133 | { |
134 | BUG_ON(!domain); | |
135 | ||
136 | domain->handler = handler; | |
77ca2332 | 137 | domain->handler_token = token; |
4f3f8d9d | 138 | } |
30bd918c | 139 | EXPORT_SYMBOL_GPL(iommu_set_fault_handler); |
4f3f8d9d | 140 | |
905d66c1 | 141 | struct iommu_domain *iommu_domain_alloc(struct bus_type *bus) |
fc2100eb JR |
142 | { |
143 | struct iommu_domain *domain; | |
144 | int ret; | |
145 | ||
94441c3b | 146 | if (bus == NULL || bus->iommu_ops == NULL) |
905d66c1 JR |
147 | return NULL; |
148 | ||
8bd6960c | 149 | domain = kzalloc(sizeof(*domain), GFP_KERNEL); |
fc2100eb JR |
150 | if (!domain) |
151 | return NULL; | |
152 | ||
94441c3b | 153 | domain->ops = bus->iommu_ops; |
905d66c1 | 154 | |
94441c3b | 155 | ret = domain->ops->domain_init(domain); |
fc2100eb JR |
156 | if (ret) |
157 | goto out_free; | |
158 | ||
159 | return domain; | |
160 | ||
161 | out_free: | |
162 | kfree(domain); | |
163 | ||
164 | return NULL; | |
165 | } | |
166 | EXPORT_SYMBOL_GPL(iommu_domain_alloc); | |
167 | ||
168 | void iommu_domain_free(struct iommu_domain *domain) | |
169 | { | |
e5aa7f00 JR |
170 | if (likely(domain->ops->domain_destroy != NULL)) |
171 | domain->ops->domain_destroy(domain); | |
172 | ||
fc2100eb JR |
173 | kfree(domain); |
174 | } | |
175 | EXPORT_SYMBOL_GPL(iommu_domain_free); | |
176 | ||
177 | int iommu_attach_device(struct iommu_domain *domain, struct device *dev) | |
178 | { | |
e5aa7f00 JR |
179 | if (unlikely(domain->ops->attach_dev == NULL)) |
180 | return -ENODEV; | |
181 | ||
182 | return domain->ops->attach_dev(domain, dev); | |
fc2100eb JR |
183 | } |
184 | EXPORT_SYMBOL_GPL(iommu_attach_device); | |
185 | ||
186 | void iommu_detach_device(struct iommu_domain *domain, struct device *dev) | |
187 | { | |
e5aa7f00 JR |
188 | if (unlikely(domain->ops->detach_dev == NULL)) |
189 | return; | |
190 | ||
191 | domain->ops->detach_dev(domain, dev); | |
fc2100eb JR |
192 | } |
193 | EXPORT_SYMBOL_GPL(iommu_detach_device); | |
194 | ||
fc2100eb JR |
195 | phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, |
196 | unsigned long iova) | |
197 | { | |
e5aa7f00 JR |
198 | if (unlikely(domain->ops->iova_to_phys == NULL)) |
199 | return 0; | |
200 | ||
201 | return domain->ops->iova_to_phys(domain, iova); | |
fc2100eb JR |
202 | } |
203 | EXPORT_SYMBOL_GPL(iommu_iova_to_phys); | |
dbb9fd86 SY |
204 | |
205 | int iommu_domain_has_cap(struct iommu_domain *domain, | |
206 | unsigned long cap) | |
207 | { | |
e5aa7f00 JR |
208 | if (unlikely(domain->ops->domain_has_cap == NULL)) |
209 | return 0; | |
210 | ||
211 | return domain->ops->domain_has_cap(domain, cap); | |
dbb9fd86 SY |
212 | } |
213 | EXPORT_SYMBOL_GPL(iommu_domain_has_cap); | |
cefc53c7 JR |
214 | |
215 | int iommu_map(struct iommu_domain *domain, unsigned long iova, | |
7d3002cc | 216 | phys_addr_t paddr, size_t size, int prot) |
cefc53c7 | 217 | { |
7d3002cc OBC |
218 | unsigned long orig_iova = iova; |
219 | unsigned int min_pagesz; | |
220 | size_t orig_size = size; | |
221 | int ret = 0; | |
cefc53c7 | 222 | |
e5aa7f00 JR |
223 | if (unlikely(domain->ops->map == NULL)) |
224 | return -ENODEV; | |
cefc53c7 | 225 | |
7d3002cc OBC |
226 | /* find out the minimum page size supported */ |
227 | min_pagesz = 1 << __ffs(domain->ops->pgsize_bitmap); | |
228 | ||
229 | /* | |
230 | * both the virtual address and the physical one, as well as | |
231 | * the size of the mapping, must be aligned (at least) to the | |
232 | * size of the smallest page supported by the hardware | |
233 | */ | |
234 | if (!IS_ALIGNED(iova | paddr | size, min_pagesz)) { | |
235 | pr_err("unaligned: iova 0x%lx pa 0x%lx size 0x%lx min_pagesz " | |
236 | "0x%x\n", iova, (unsigned long)paddr, | |
237 | (unsigned long)size, min_pagesz); | |
238 | return -EINVAL; | |
239 | } | |
240 | ||
241 | pr_debug("map: iova 0x%lx pa 0x%lx size 0x%lx\n", iova, | |
242 | (unsigned long)paddr, (unsigned long)size); | |
243 | ||
244 | while (size) { | |
245 | unsigned long pgsize, addr_merge = iova | paddr; | |
246 | unsigned int pgsize_idx; | |
247 | ||
248 | /* Max page size that still fits into 'size' */ | |
249 | pgsize_idx = __fls(size); | |
250 | ||
251 | /* need to consider alignment requirements ? */ | |
252 | if (likely(addr_merge)) { | |
253 | /* Max page size allowed by both iova and paddr */ | |
254 | unsigned int align_pgsize_idx = __ffs(addr_merge); | |
255 | ||
256 | pgsize_idx = min(pgsize_idx, align_pgsize_idx); | |
257 | } | |
258 | ||
259 | /* build a mask of acceptable page sizes */ | |
260 | pgsize = (1UL << (pgsize_idx + 1)) - 1; | |
261 | ||
262 | /* throw away page sizes not supported by the hardware */ | |
263 | pgsize &= domain->ops->pgsize_bitmap; | |
cefc53c7 | 264 | |
7d3002cc OBC |
265 | /* make sure we're still sane */ |
266 | BUG_ON(!pgsize); | |
cefc53c7 | 267 | |
7d3002cc OBC |
268 | /* pick the biggest page */ |
269 | pgsize_idx = __fls(pgsize); | |
270 | pgsize = 1UL << pgsize_idx; | |
271 | ||
272 | pr_debug("mapping: iova 0x%lx pa 0x%lx pgsize %lu\n", iova, | |
273 | (unsigned long)paddr, pgsize); | |
274 | ||
275 | ret = domain->ops->map(domain, iova, paddr, pgsize, prot); | |
276 | if (ret) | |
277 | break; | |
278 | ||
279 | iova += pgsize; | |
280 | paddr += pgsize; | |
281 | size -= pgsize; | |
282 | } | |
283 | ||
284 | /* unroll mapping in case something went wrong */ | |
285 | if (ret) | |
286 | iommu_unmap(domain, orig_iova, orig_size - size); | |
287 | ||
288 | return ret; | |
cefc53c7 JR |
289 | } |
290 | EXPORT_SYMBOL_GPL(iommu_map); | |
291 | ||
7d3002cc | 292 | size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size) |
cefc53c7 | 293 | { |
7d3002cc OBC |
294 | size_t unmapped_page, unmapped = 0; |
295 | unsigned int min_pagesz; | |
cefc53c7 | 296 | |
e5aa7f00 JR |
297 | if (unlikely(domain->ops->unmap == NULL)) |
298 | return -ENODEV; | |
299 | ||
7d3002cc OBC |
300 | /* find out the minimum page size supported */ |
301 | min_pagesz = 1 << __ffs(domain->ops->pgsize_bitmap); | |
302 | ||
303 | /* | |
304 | * The virtual address, as well as the size of the mapping, must be | |
305 | * aligned (at least) to the size of the smallest page supported | |
306 | * by the hardware | |
307 | */ | |
308 | if (!IS_ALIGNED(iova | size, min_pagesz)) { | |
309 | pr_err("unaligned: iova 0x%lx size 0x%lx min_pagesz 0x%x\n", | |
310 | iova, (unsigned long)size, min_pagesz); | |
311 | return -EINVAL; | |
312 | } | |
313 | ||
314 | pr_debug("unmap this: iova 0x%lx size 0x%lx\n", iova, | |
315 | (unsigned long)size); | |
316 | ||
317 | /* | |
318 | * Keep iterating until we either unmap 'size' bytes (or more) | |
319 | * or we hit an area that isn't mapped. | |
320 | */ | |
321 | while (unmapped < size) { | |
322 | size_t left = size - unmapped; | |
323 | ||
324 | unmapped_page = domain->ops->unmap(domain, iova, left); | |
325 | if (!unmapped_page) | |
326 | break; | |
327 | ||
328 | pr_debug("unmapped: iova 0x%lx size %lx\n", iova, | |
329 | (unsigned long)unmapped_page); | |
330 | ||
331 | iova += unmapped_page; | |
332 | unmapped += unmapped_page; | |
333 | } | |
334 | ||
335 | return unmapped; | |
cefc53c7 JR |
336 | } |
337 | EXPORT_SYMBOL_GPL(iommu_unmap); | |
1460432c AW |
338 | |
339 | int iommu_device_group(struct device *dev, unsigned int *groupid) | |
340 | { | |
341 | if (iommu_present(dev->bus) && dev->bus->iommu_ops->device_group) | |
342 | return dev->bus->iommu_ops->device_group(dev, groupid); | |
343 | ||
344 | return -ENODEV; | |
345 | } | |
346 | EXPORT_SYMBOL_GPL(iommu_device_group); |