]>
Commit | Line | Data |
---|---|---|
318f67ce AK |
1 | /* |
2 | * DMA memory preregistration | |
3 | * | |
4 | * Authors: | |
5 | * Alexey Kardashevskiy <aik@ozlabs.ru> | |
6 | * | |
7 | * This work is licensed under the terms of the GNU GPL, version 2. See | |
8 | * the COPYING file in the top-level directory. | |
9 | */ | |
10 | ||
11 | #include "qemu/osdep.h" | |
318f67ce AK |
12 | #include <sys/ioctl.h> |
13 | #include <linux/vfio.h> | |
521c8f4e ZD |
14 | #ifdef CONFIG_KVM |
15 | #include <linux/kvm.h> | |
16 | #endif | |
17 | #include "sysemu/kvm.h" | |
770c3b6e | 18 | #include "exec/address-spaces.h" |
318f67ce AK |
19 | |
20 | #include "hw/vfio/vfio-common.h" | |
21 | #include "hw/hw.h" | |
c26bc185 | 22 | #include "exec/ram_addr.h" |
318f67ce | 23 | #include "qemu/error-report.h" |
d7d87836 | 24 | #include "qapi/error.h" |
318f67ce AK |
25 | #include "trace.h" |
26 | ||
27 | static bool vfio_prereg_listener_skipped_section(MemoryRegionSection *section) | |
28 | { | |
29 | if (memory_region_is_iommu(section->mr)) { | |
30 | hw_error("Cannot possibly preregister IOMMU memory"); | |
31 | } | |
32 | ||
33 | return !memory_region_is_ram(section->mr) || | |
21e00fa5 | 34 | memory_region_is_ram_device(section->mr); |
318f67ce AK |
35 | } |
36 | ||
37 | static void *vfio_prereg_gpa_to_vaddr(MemoryRegionSection *section, hwaddr gpa) | |
38 | { | |
39 | return memory_region_get_ram_ptr(section->mr) + | |
40 | section->offset_within_region + | |
41 | (gpa - section->offset_within_address_space); | |
42 | } | |
43 | ||
44 | static void vfio_prereg_listener_region_add(MemoryListener *listener, | |
45 | MemoryRegionSection *section) | |
46 | { | |
47 | VFIOContainer *container = container_of(listener, VFIOContainer, | |
48 | prereg_listener); | |
49 | const hwaddr gpa = section->offset_within_address_space; | |
50 | hwaddr end; | |
51 | int ret; | |
8e3b0cbb | 52 | hwaddr page_mask = qemu_real_host_page_mask(); |
318f67ce AK |
53 | struct vfio_iommu_spapr_register_memory reg = { |
54 | .argsz = sizeof(reg), | |
55 | .flags = 0, | |
56 | }; | |
57 | ||
58 | if (vfio_prereg_listener_skipped_section(section)) { | |
59 | trace_vfio_prereg_listener_region_add_skip( | |
60 | section->offset_within_address_space, | |
61 | section->offset_within_address_space + | |
62 | int128_get64(int128_sub(section->size, int128_one()))); | |
63 | return; | |
64 | } | |
65 | ||
66 | if (unlikely((section->offset_within_address_space & ~page_mask) || | |
67 | (section->offset_within_region & ~page_mask) || | |
68 | (int128_get64(section->size) & ~page_mask))) { | |
69 | error_report("%s received unaligned region", __func__); | |
70 | return; | |
71 | } | |
72 | ||
73 | end = section->offset_within_address_space + int128_get64(section->size); | |
74 | if (gpa >= end) { | |
75 | return; | |
76 | } | |
77 | ||
78 | memory_region_ref(section->mr); | |
79 | ||
80 | reg.vaddr = (uintptr_t) vfio_prereg_gpa_to_vaddr(section, gpa); | |
81 | reg.size = end - gpa; | |
82 | ||
83 | ret = ioctl(container->fd, VFIO_IOMMU_SPAPR_REGISTER_MEMORY, ®); | |
84 | trace_vfio_prereg_register(reg.vaddr, reg.size, ret ? -errno : 0); | |
85 | if (ret) { | |
86 | /* | |
87 | * On the initfn path, store the first error in the container so we | |
88 | * can gracefully fail. Runtime, there's not much we can do other | |
89 | * than throw a hardware error. | |
90 | */ | |
91 | if (!container->initialized) { | |
92 | if (!container->error) { | |
d7d87836 EA |
93 | error_setg_errno(&container->error, -ret, |
94 | "Memory registering failed"); | |
318f67ce AK |
95 | } |
96 | } else { | |
97 | hw_error("vfio: Memory registering failed, unable to continue"); | |
98 | } | |
99 | } | |
100 | } | |
101 | ||
102 | static void vfio_prereg_listener_region_del(MemoryListener *listener, | |
103 | MemoryRegionSection *section) | |
104 | { | |
105 | VFIOContainer *container = container_of(listener, VFIOContainer, | |
106 | prereg_listener); | |
107 | const hwaddr gpa = section->offset_within_address_space; | |
108 | hwaddr end; | |
109 | int ret; | |
8e3b0cbb | 110 | hwaddr page_mask = qemu_real_host_page_mask(); |
318f67ce AK |
111 | struct vfio_iommu_spapr_register_memory reg = { |
112 | .argsz = sizeof(reg), | |
113 | .flags = 0, | |
114 | }; | |
115 | ||
116 | if (vfio_prereg_listener_skipped_section(section)) { | |
117 | trace_vfio_prereg_listener_region_del_skip( | |
118 | section->offset_within_address_space, | |
119 | section->offset_within_address_space + | |
120 | int128_get64(int128_sub(section->size, int128_one()))); | |
121 | return; | |
122 | } | |
123 | ||
124 | if (unlikely((section->offset_within_address_space & ~page_mask) || | |
125 | (section->offset_within_region & ~page_mask) || | |
126 | (int128_get64(section->size) & ~page_mask))) { | |
127 | error_report("%s received unaligned region", __func__); | |
128 | return; | |
129 | } | |
130 | ||
131 | end = section->offset_within_address_space + int128_get64(section->size); | |
132 | if (gpa >= end) { | |
133 | return; | |
134 | } | |
135 | ||
136 | reg.vaddr = (uintptr_t) vfio_prereg_gpa_to_vaddr(section, gpa); | |
137 | reg.size = end - gpa; | |
138 | ||
139 | ret = ioctl(container->fd, VFIO_IOMMU_SPAPR_UNREGISTER_MEMORY, ®); | |
140 | trace_vfio_prereg_unregister(reg.vaddr, reg.size, ret ? -errno : 0); | |
141 | } | |
142 | ||
770c3b6e | 143 | static const MemoryListener vfio_prereg_listener = { |
142518bd | 144 | .name = "vfio-pre-reg", |
318f67ce AK |
145 | .region_add = vfio_prereg_listener_region_add, |
146 | .region_del = vfio_prereg_listener_region_del, | |
147 | }; | |
2e4109de | 148 | |
a2347c60 ZD |
149 | static void vfio_host_win_add(VFIOContainer *container, hwaddr min_iova, |
150 | hwaddr max_iova, uint64_t iova_pgsizes) | |
151 | { | |
152 | VFIOHostDMAWindow *hostwin; | |
153 | ||
154 | QLIST_FOREACH(hostwin, &container->hostwin_list, hostwin_next) { | |
155 | if (ranges_overlap(hostwin->min_iova, | |
156 | hostwin->max_iova - hostwin->min_iova + 1, | |
157 | min_iova, | |
158 | max_iova - min_iova + 1)) { | |
159 | hw_error("%s: Overlapped IOMMU are not enabled", __func__); | |
160 | } | |
161 | } | |
162 | ||
163 | hostwin = g_malloc0(sizeof(*hostwin)); | |
164 | ||
165 | hostwin->min_iova = min_iova; | |
166 | hostwin->max_iova = max_iova; | |
167 | hostwin->iova_pgsizes = iova_pgsizes; | |
168 | QLIST_INSERT_HEAD(&container->hostwin_list, hostwin, hostwin_next); | |
169 | } | |
170 | ||
171 | static int vfio_host_win_del(VFIOContainer *container, | |
172 | hwaddr min_iova, hwaddr max_iova) | |
173 | { | |
174 | VFIOHostDMAWindow *hostwin; | |
175 | ||
176 | QLIST_FOREACH(hostwin, &container->hostwin_list, hostwin_next) { | |
177 | if (hostwin->min_iova == min_iova && hostwin->max_iova == max_iova) { | |
178 | QLIST_REMOVE(hostwin, hostwin_next); | |
179 | g_free(hostwin); | |
180 | return 0; | |
181 | } | |
182 | } | |
183 | ||
184 | return -1; | |
185 | } | |
186 | ||
187 | static VFIOHostDMAWindow *vfio_find_hostwin(VFIOContainer *container, | |
188 | hwaddr iova, hwaddr end) | |
189 | { | |
190 | VFIOHostDMAWindow *hostwin; | |
191 | bool hostwin_found = false; | |
192 | ||
193 | QLIST_FOREACH(hostwin, &container->hostwin_list, hostwin_next) { | |
194 | if (hostwin->min_iova <= iova && end <= hostwin->max_iova) { | |
195 | hostwin_found = true; | |
196 | break; | |
197 | } | |
198 | } | |
199 | ||
200 | return hostwin_found ? hostwin : NULL; | |
201 | } | |
202 | ||
a17879f0 ZD |
203 | static int vfio_spapr_remove_window(VFIOContainer *container, |
204 | hwaddr offset_within_address_space) | |
205 | { | |
206 | struct vfio_iommu_spapr_tce_remove remove = { | |
207 | .argsz = sizeof(remove), | |
208 | .start_addr = offset_within_address_space, | |
209 | }; | |
210 | int ret; | |
211 | ||
212 | ret = ioctl(container->fd, VFIO_IOMMU_SPAPR_TCE_REMOVE, &remove); | |
213 | if (ret) { | |
214 | error_report("Failed to remove window at %"PRIx64, | |
215 | (uint64_t)remove.start_addr); | |
216 | return -errno; | |
217 | } | |
218 | ||
219 | trace_vfio_spapr_remove_window(offset_within_address_space); | |
220 | ||
221 | return 0; | |
222 | } | |
223 | ||
224 | static int vfio_spapr_create_window(VFIOContainer *container, | |
225 | MemoryRegionSection *section, | |
226 | hwaddr *pgsize) | |
2e4109de | 227 | { |
16107998 | 228 | int ret = 0; |
7ab1cb74 | 229 | VFIOContainerBase *bcontainer = &container->bcontainer; |
3df9d748 | 230 | IOMMUMemoryRegion *iommu_mr = IOMMU_MEMORY_REGION(section->mr); |
79178edd | 231 | uint64_t pagesize = memory_region_iommu_get_min_page_size(iommu_mr), pgmask; |
16107998 | 232 | unsigned entries, bits_total, bits_per_level, max_levels; |
2e4109de | 233 | struct vfio_iommu_spapr_tce_create create = { .argsz = sizeof(create) }; |
905b7ee4 | 234 | long rampagesize = qemu_minrampagesize(); |
c26bc185 AK |
235 | |
236 | /* | |
237 | * The host might not support the guest supported IOMMU page size, | |
238 | * so we will use smaller physical IOMMU pages to back them. | |
239 | */ | |
3cdd801b AK |
240 | if (pagesize > rampagesize) { |
241 | pagesize = rampagesize; | |
c26bc185 | 242 | } |
7ab1cb74 | 243 | pgmask = bcontainer->pgsizes & (pagesize | (pagesize - 1)); |
79178edd | 244 | pagesize = pgmask ? (1ULL << (63 - clz64(pgmask))) : 0; |
c26bc185 AK |
245 | if (!pagesize) { |
246 | error_report("Host doesn't support page size 0x%"PRIx64 | |
247 | ", the supported mask is 0x%lx", | |
248 | memory_region_iommu_get_min_page_size(iommu_mr), | |
7ab1cb74 | 249 | bcontainer->pgsizes); |
c26bc185 AK |
250 | return -EINVAL; |
251 | } | |
2e4109de AK |
252 | |
253 | /* | |
254 | * FIXME: For VFIO iommu types which have KVM acceleration to | |
255 | * avoid bouncing all map/unmaps through qemu this way, this | |
256 | * would be the right place to wire that up (tell the KVM | |
257 | * device emulation the VFIO iommu handles to use). | |
258 | */ | |
259 | create.window_size = int128_get64(section->size); | |
260 | create.page_shift = ctz64(pagesize); | |
261 | /* | |
16107998 AK |
262 | * SPAPR host supports multilevel TCE tables. We try to guess optimal |
263 | * levels number and if this fails (for example due to the host memory | |
264 | * fragmentation), we increase levels. The DMA address structure is: | |
265 | * rrrrrrrr rxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx iiiiiiii | |
266 | * where: | |
267 | * r = reserved (bits >= 55 are reserved in the existing hardware) | |
268 | * i = IOMMU page offset (64K in this example) | |
269 | * x = bits to index a TCE which can be split to equal chunks to index | |
270 | * within the level. | |
271 | * The aim is to split "x" to smaller possible number of levels. | |
2e4109de AK |
272 | */ |
273 | entries = create.window_size >> create.page_shift; | |
16107998 AK |
274 | /* bits_total is number of "x" needed */ |
275 | bits_total = ctz64(entries * sizeof(uint64_t)); | |
276 | /* | |
277 | * bits_per_level is a safe guess of how much we can allocate per level: | |
278 | * 8 is the current minimum for CONFIG_FORCE_MAX_ZONEORDER and MAX_ORDER | |
279 | * is usually bigger than that. | |
038adc2f WY |
280 | * Below we look at qemu_real_host_page_size as TCEs are allocated from |
281 | * system pages. | |
16107998 | 282 | */ |
8e3b0cbb | 283 | bits_per_level = ctz64(qemu_real_host_page_size()) + 8; |
16107998 AK |
284 | create.levels = bits_total / bits_per_level; |
285 | if (bits_total % bits_per_level) { | |
286 | ++create.levels; | |
287 | } | |
8e3b0cbb | 288 | max_levels = (64 - create.page_shift) / ctz64(qemu_real_host_page_size()); |
16107998 AK |
289 | for ( ; create.levels <= max_levels; ++create.levels) { |
290 | ret = ioctl(container->fd, VFIO_IOMMU_SPAPR_TCE_CREATE, &create); | |
291 | if (!ret) { | |
292 | break; | |
293 | } | |
294 | } | |
2e4109de AK |
295 | if (ret) { |
296 | error_report("Failed to create a window, ret = %d (%m)", ret); | |
297 | return -errno; | |
298 | } | |
299 | ||
300 | if (create.start_addr != section->offset_within_address_space) { | |
301 | vfio_spapr_remove_window(container, create.start_addr); | |
302 | ||
303 | error_report("Host doesn't support DMA window at %"HWADDR_PRIx", must be %"PRIx64, | |
304 | section->offset_within_address_space, | |
305 | (uint64_t)create.start_addr); | |
2e4109de AK |
306 | return -EINVAL; |
307 | } | |
308 | trace_vfio_spapr_create_window(create.page_shift, | |
16107998 | 309 | create.levels, |
2e4109de AK |
310 | create.window_size, |
311 | create.start_addr); | |
312 | *pgsize = pagesize; | |
313 | ||
314 | return 0; | |
315 | } | |
316 | ||
521c8f4e ZD |
317 | int vfio_container_add_section_window(VFIOContainer *container, |
318 | MemoryRegionSection *section, | |
319 | Error **errp) | |
320 | { | |
321 | VFIOHostDMAWindow *hostwin; | |
322 | hwaddr pgsize = 0; | |
323 | int ret; | |
324 | ||
a2347c60 ZD |
325 | /* |
326 | * VFIO_SPAPR_TCE_IOMMU supports a single host window between | |
327 | * [dma32_window_start, dma32_window_size), we need to ensure | |
328 | * the section fall in this range. | |
329 | */ | |
330 | if (container->iommu_type == VFIO_SPAPR_TCE_IOMMU) { | |
331 | hwaddr iova, end; | |
332 | ||
333 | iova = section->offset_within_address_space; | |
334 | end = iova + int128_get64(section->size) - 1; | |
335 | ||
336 | if (!vfio_find_hostwin(container, iova, end)) { | |
337 | error_setg(errp, "Container %p can't map guest IOVA region" | |
338 | " 0x%"HWADDR_PRIx"..0x%"HWADDR_PRIx, container, | |
339 | iova, end); | |
340 | return -EINVAL; | |
341 | } | |
342 | return 0; | |
343 | } | |
344 | ||
521c8f4e ZD |
345 | if (container->iommu_type != VFIO_SPAPR_TCE_v2_IOMMU) { |
346 | return 0; | |
347 | } | |
348 | ||
349 | /* For now intersections are not allowed, we may relax this later */ | |
350 | QLIST_FOREACH(hostwin, &container->hostwin_list, hostwin_next) { | |
351 | if (ranges_overlap(hostwin->min_iova, | |
352 | hostwin->max_iova - hostwin->min_iova + 1, | |
353 | section->offset_within_address_space, | |
354 | int128_get64(section->size))) { | |
355 | error_setg(errp, | |
356 | "region [0x%"PRIx64",0x%"PRIx64"] overlaps with existing" | |
357 | "host DMA window [0x%"PRIx64",0x%"PRIx64"]", | |
358 | section->offset_within_address_space, | |
359 | section->offset_within_address_space + | |
360 | int128_get64(section->size) - 1, | |
361 | hostwin->min_iova, hostwin->max_iova); | |
362 | return -EINVAL; | |
363 | } | |
364 | } | |
365 | ||
366 | ret = vfio_spapr_create_window(container, section, &pgsize); | |
367 | if (ret) { | |
368 | error_setg_errno(errp, -ret, "Failed to create SPAPR window"); | |
369 | return ret; | |
370 | } | |
371 | ||
372 | vfio_host_win_add(container, section->offset_within_address_space, | |
373 | section->offset_within_address_space + | |
374 | int128_get64(section->size) - 1, pgsize); | |
375 | #ifdef CONFIG_KVM | |
376 | if (kvm_enabled()) { | |
377 | VFIOGroup *group; | |
378 | IOMMUMemoryRegion *iommu_mr = IOMMU_MEMORY_REGION(section->mr); | |
379 | struct kvm_vfio_spapr_tce param; | |
380 | struct kvm_device_attr attr = { | |
381 | .group = KVM_DEV_VFIO_GROUP, | |
382 | .attr = KVM_DEV_VFIO_GROUP_SET_SPAPR_TCE, | |
383 | .addr = (uint64_t)(unsigned long)¶m, | |
384 | }; | |
385 | ||
386 | if (!memory_region_iommu_get_attr(iommu_mr, IOMMU_ATTR_SPAPR_TCE_FD, | |
387 | ¶m.tablefd)) { | |
388 | QLIST_FOREACH(group, &container->group_list, container_next) { | |
389 | param.groupfd = group->fd; | |
390 | if (ioctl(vfio_kvm_device_fd, KVM_SET_DEVICE_ATTR, &attr)) { | |
391 | error_setg_errno(errp, errno, | |
392 | "vfio: failed GROUP_SET_SPAPR_TCE for " | |
393 | "KVM VFIO device %d and group fd %d", | |
394 | param.tablefd, param.groupfd); | |
395 | return -errno; | |
396 | } | |
397 | trace_vfio_spapr_group_attach(param.groupfd, param.tablefd); | |
398 | } | |
399 | } | |
400 | } | |
401 | #endif | |
402 | return 0; | |
403 | } | |
404 | ||
405 | void vfio_container_del_section_window(VFIOContainer *container, | |
406 | MemoryRegionSection *section) | |
407 | { | |
408 | if (container->iommu_type != VFIO_SPAPR_TCE_v2_IOMMU) { | |
409 | return; | |
410 | } | |
411 | ||
412 | vfio_spapr_remove_window(container, | |
413 | section->offset_within_address_space); | |
414 | if (vfio_host_win_del(container, | |
415 | section->offset_within_address_space, | |
416 | section->offset_within_address_space + | |
417 | int128_get64(section->size) - 1) < 0) { | |
418 | hw_error("%s: Cannot delete missing window at %"HWADDR_PRIx, | |
419 | __func__, section->offset_within_address_space); | |
420 | } | |
421 | } | |
770c3b6e ZD |
422 | |
423 | int vfio_spapr_container_init(VFIOContainer *container, Error **errp) | |
424 | { | |
7ab1cb74 | 425 | VFIOContainerBase *bcontainer = &container->bcontainer; |
770c3b6e ZD |
426 | struct vfio_iommu_spapr_tce_info info; |
427 | bool v2 = container->iommu_type == VFIO_SPAPR_TCE_v2_IOMMU; | |
428 | int ret, fd = container->fd; | |
429 | ||
a2347c60 ZD |
430 | QLIST_INIT(&container->hostwin_list); |
431 | ||
770c3b6e ZD |
432 | /* |
433 | * The host kernel code implementing VFIO_IOMMU_DISABLE is called | |
434 | * when container fd is closed so we do not call it explicitly | |
435 | * in this file. | |
436 | */ | |
437 | if (!v2) { | |
438 | ret = ioctl(fd, VFIO_IOMMU_ENABLE); | |
439 | if (ret) { | |
440 | error_setg_errno(errp, errno, "failed to enable container"); | |
441 | return -errno; | |
442 | } | |
443 | } else { | |
444 | container->prereg_listener = vfio_prereg_listener; | |
445 | ||
446 | memory_listener_register(&container->prereg_listener, | |
447 | &address_space_memory); | |
448 | if (container->error) { | |
449 | ret = -1; | |
450 | error_propagate_prepend(errp, container->error, | |
451 | "RAM memory listener initialization failed: "); | |
452 | goto listener_unregister_exit; | |
453 | } | |
454 | } | |
455 | ||
456 | info.argsz = sizeof(info); | |
457 | ret = ioctl(fd, VFIO_IOMMU_SPAPR_TCE_GET_INFO, &info); | |
458 | if (ret) { | |
459 | error_setg_errno(errp, errno, | |
460 | "VFIO_IOMMU_SPAPR_TCE_GET_INFO failed"); | |
461 | ret = -errno; | |
462 | goto listener_unregister_exit; | |
463 | } | |
464 | ||
465 | if (v2) { | |
7ab1cb74 | 466 | bcontainer->pgsizes = info.ddw.pgsizes; |
770c3b6e ZD |
467 | /* |
468 | * There is a default window in just created container. | |
469 | * To make region_add/del simpler, we better remove this | |
470 | * window now and let those iommu_listener callbacks | |
471 | * create/remove them when needed. | |
472 | */ | |
473 | ret = vfio_spapr_remove_window(container, info.dma32_window_start); | |
474 | if (ret) { | |
475 | error_setg_errno(errp, -ret, | |
476 | "failed to remove existing window"); | |
477 | goto listener_unregister_exit; | |
478 | } | |
479 | } else { | |
480 | /* The default table uses 4K pages */ | |
7ab1cb74 | 481 | bcontainer->pgsizes = 0x1000; |
770c3b6e ZD |
482 | vfio_host_win_add(container, info.dma32_window_start, |
483 | info.dma32_window_start + | |
484 | info.dma32_window_size - 1, | |
485 | 0x1000); | |
486 | } | |
487 | ||
488 | return 0; | |
489 | ||
490 | listener_unregister_exit: | |
491 | if (v2) { | |
492 | memory_listener_unregister(&container->prereg_listener); | |
493 | } | |
494 | return ret; | |
495 | } | |
496 | ||
497 | void vfio_spapr_container_deinit(VFIOContainer *container) | |
498 | { | |
a2347c60 ZD |
499 | VFIOHostDMAWindow *hostwin, *next; |
500 | ||
770c3b6e ZD |
501 | if (container->iommu_type == VFIO_SPAPR_TCE_v2_IOMMU) { |
502 | memory_listener_unregister(&container->prereg_listener); | |
503 | } | |
a2347c60 ZD |
504 | QLIST_FOREACH_SAFE(hostwin, &container->hostwin_list, hostwin_next, |
505 | next) { | |
506 | QLIST_REMOVE(hostwin, hostwin_next); | |
507 | g_free(hostwin); | |
508 | } | |
770c3b6e | 509 | } |