]>
Commit | Line | Data |
---|---|---|
22c37a10 EA |
1 | /* |
2 | * virtio-iommu device | |
3 | * | |
4 | * Copyright (c) 2020 Red Hat, Inc. | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or modify it | |
7 | * under the terms and conditions of the GNU General Public License, | |
8 | * version 2 or later, as published by the Free Software Foundation. | |
9 | * | |
10 | * This program is distributed in the hope it will be useful, but WITHOUT | |
11 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
12 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for | |
13 | * more details. | |
14 | * | |
15 | * You should have received a copy of the GNU General Public License along with | |
16 | * this program. If not, see <http://www.gnu.org/licenses/>. | |
17 | * | |
18 | */ | |
19 | ||
20 | #include "qemu/osdep.h" | |
fe2cacae | 21 | #include "qemu/log.h" |
22c37a10 | 22 | #include "qemu/iov.h" |
22c37a10 EA |
23 | #include "hw/qdev-properties.h" |
24 | #include "hw/virtio/virtio.h" | |
25 | #include "sysemu/kvm.h" | |
448179e3 | 26 | #include "sysemu/reset.h" |
cfb42188 EA |
27 | #include "qapi/error.h" |
28 | #include "qemu/error-report.h" | |
22c37a10 EA |
29 | #include "trace.h" |
30 | ||
31 | #include "standard-headers/linux/virtio_ids.h" | |
32 | ||
33 | #include "hw/virtio/virtio-bus.h" | |
34 | #include "hw/virtio/virtio-access.h" | |
35 | #include "hw/virtio/virtio-iommu.h" | |
cfb42188 EA |
36 | #include "hw/pci/pci_bus.h" |
37 | #include "hw/pci/pci.h" | |
22c37a10 EA |
38 | |
39 | /* Max size */ | |
40 | #define VIOMMU_DEFAULT_QUEUE_SIZE 256 | |
1733eebb | 41 | #define VIOMMU_PROBE_SIZE 512 |
22c37a10 | 42 | |
cfb42188 EA |
43 | typedef struct VirtIOIOMMUDomain { |
44 | uint32_t id; | |
d9c96f24 | 45 | bool bypass; |
cfb42188 EA |
46 | GTree *mappings; |
47 | QLIST_HEAD(, VirtIOIOMMUEndpoint) endpoint_list; | |
48 | } VirtIOIOMMUDomain; | |
49 | ||
50 | typedef struct VirtIOIOMMUEndpoint { | |
51 | uint32_t id; | |
52 | VirtIOIOMMUDomain *domain; | |
31aa323f | 53 | IOMMUMemoryRegion *iommu_mr; |
cfb42188 EA |
54 | QLIST_ENTRY(VirtIOIOMMUEndpoint) next; |
55 | } VirtIOIOMMUEndpoint; | |
56 | ||
57 | typedef struct VirtIOIOMMUInterval { | |
58 | uint64_t low; | |
59 | uint64_t high; | |
60 | } VirtIOIOMMUInterval; | |
61 | ||
fe2cacae EA |
62 | typedef struct VirtIOIOMMUMapping { |
63 | uint64_t phys_addr; | |
64 | uint32_t flags; | |
65 | } VirtIOIOMMUMapping; | |
66 | ||
cfb42188 EA |
67 | static inline uint16_t virtio_iommu_get_bdf(IOMMUDevice *dev) |
68 | { | |
69 | return PCI_BUILD_BDF(pci_bus_num(dev->bus), dev->devfn); | |
70 | } | |
71 | ||
90519b90 ZD |
72 | static bool virtio_iommu_device_bypassed(IOMMUDevice *sdev) |
73 | { | |
74 | uint32_t sid; | |
75 | bool bypassed; | |
76 | VirtIOIOMMU *s = sdev->viommu; | |
77 | VirtIOIOMMUEndpoint *ep; | |
78 | ||
79 | sid = virtio_iommu_get_bdf(sdev); | |
80 | ||
08f2030a | 81 | qemu_rec_mutex_lock(&s->mutex); |
90519b90 ZD |
82 | /* need to check bypass before system reset */ |
83 | if (!s->endpoints) { | |
84 | bypassed = s->config.bypass; | |
85 | goto unlock; | |
86 | } | |
87 | ||
88 | ep = g_tree_lookup(s->endpoints, GUINT_TO_POINTER(sid)); | |
89 | if (!ep || !ep->domain) { | |
90 | bypassed = s->config.bypass; | |
91 | } else { | |
92 | bypassed = ep->domain->bypass; | |
93 | } | |
94 | ||
95 | unlock: | |
08f2030a | 96 | qemu_rec_mutex_unlock(&s->mutex); |
90519b90 ZD |
97 | return bypassed; |
98 | } | |
99 | ||
100 | /* Return whether the device is using IOMMU translation. */ | |
101 | static bool virtio_iommu_switch_address_space(IOMMUDevice *sdev) | |
102 | { | |
103 | bool use_remapping; | |
104 | ||
105 | assert(sdev); | |
106 | ||
107 | use_remapping = !virtio_iommu_device_bypassed(sdev); | |
108 | ||
109 | trace_virtio_iommu_switch_address_space(pci_bus_num(sdev->bus), | |
110 | PCI_SLOT(sdev->devfn), | |
111 | PCI_FUNC(sdev->devfn), | |
112 | use_remapping); | |
113 | ||
114 | /* Turn off first then on the other */ | |
115 | if (use_remapping) { | |
116 | memory_region_set_enabled(&sdev->bypass_mr, false); | |
117 | memory_region_set_enabled(MEMORY_REGION(&sdev->iommu_mr), true); | |
118 | } else { | |
119 | memory_region_set_enabled(MEMORY_REGION(&sdev->iommu_mr), false); | |
120 | memory_region_set_enabled(&sdev->bypass_mr, true); | |
121 | } | |
122 | ||
123 | return use_remapping; | |
124 | } | |
125 | ||
126 | static void virtio_iommu_switch_address_space_all(VirtIOIOMMU *s) | |
127 | { | |
128 | GHashTableIter iter; | |
129 | IOMMUPciBus *iommu_pci_bus; | |
130 | int i; | |
131 | ||
132 | g_hash_table_iter_init(&iter, s->as_by_busptr); | |
133 | while (g_hash_table_iter_next(&iter, NULL, (void **)&iommu_pci_bus)) { | |
134 | for (i = 0; i < PCI_DEVFN_MAX; i++) { | |
135 | if (!iommu_pci_bus->pbdev[i]) { | |
136 | continue; | |
137 | } | |
138 | virtio_iommu_switch_address_space(iommu_pci_bus->pbdev[i]); | |
139 | } | |
140 | } | |
141 | } | |
142 | ||
cfb42188 EA |
143 | /** |
144 | * The bus number is used for lookup when SID based operations occur. | |
145 | * In that case we lazily populate the IOMMUPciBus array from the bus hash | |
146 | * table. At the time the IOMMUPciBus is created (iommu_find_add_as), the bus | |
147 | * numbers may not be always initialized yet. | |
148 | */ | |
149 | static IOMMUPciBus *iommu_find_iommu_pcibus(VirtIOIOMMU *s, uint8_t bus_num) | |
150 | { | |
151 | IOMMUPciBus *iommu_pci_bus = s->iommu_pcibus_by_bus_num[bus_num]; | |
152 | ||
153 | if (!iommu_pci_bus) { | |
154 | GHashTableIter iter; | |
155 | ||
156 | g_hash_table_iter_init(&iter, s->as_by_busptr); | |
157 | while (g_hash_table_iter_next(&iter, NULL, (void **)&iommu_pci_bus)) { | |
158 | if (pci_bus_num(iommu_pci_bus->bus) == bus_num) { | |
159 | s->iommu_pcibus_by_bus_num[bus_num] = iommu_pci_bus; | |
160 | return iommu_pci_bus; | |
161 | } | |
162 | } | |
163 | return NULL; | |
164 | } | |
165 | return iommu_pci_bus; | |
166 | } | |
167 | ||
168 | static IOMMUMemoryRegion *virtio_iommu_mr(VirtIOIOMMU *s, uint32_t sid) | |
169 | { | |
170 | uint8_t bus_n, devfn; | |
171 | IOMMUPciBus *iommu_pci_bus; | |
172 | IOMMUDevice *dev; | |
173 | ||
174 | bus_n = PCI_BUS_NUM(sid); | |
175 | iommu_pci_bus = iommu_find_iommu_pcibus(s, bus_n); | |
176 | if (iommu_pci_bus) { | |
bfe7a961 | 177 | devfn = sid & (PCI_DEVFN_MAX - 1); |
cfb42188 EA |
178 | dev = iommu_pci_bus->pbdev[devfn]; |
179 | if (dev) { | |
180 | return &dev->iommu_mr; | |
181 | } | |
182 | } | |
183 | return NULL; | |
184 | } | |
185 | ||
186 | static gint interval_cmp(gconstpointer a, gconstpointer b, gpointer user_data) | |
187 | { | |
188 | VirtIOIOMMUInterval *inta = (VirtIOIOMMUInterval *)a; | |
189 | VirtIOIOMMUInterval *intb = (VirtIOIOMMUInterval *)b; | |
190 | ||
191 | if (inta->high < intb->low) { | |
192 | return -1; | |
193 | } else if (intb->high < inta->low) { | |
194 | return 1; | |
195 | } else { | |
196 | return 0; | |
197 | } | |
198 | } | |
199 | ||
15e4c8f0 BB |
200 | static void virtio_iommu_notify_map(IOMMUMemoryRegion *mr, hwaddr virt_start, |
201 | hwaddr virt_end, hwaddr paddr, | |
202 | uint32_t flags) | |
203 | { | |
5039caf3 | 204 | IOMMUTLBEvent event; |
15e4c8f0 BB |
205 | IOMMUAccessFlags perm = IOMMU_ACCESS_FLAG(flags & VIRTIO_IOMMU_MAP_F_READ, |
206 | flags & VIRTIO_IOMMU_MAP_F_WRITE); | |
207 | ||
208 | if (!(mr->iommu_notify_flags & IOMMU_NOTIFIER_MAP) || | |
209 | (flags & VIRTIO_IOMMU_MAP_F_MMIO) || !perm) { | |
210 | return; | |
211 | } | |
212 | ||
213 | trace_virtio_iommu_notify_map(mr->parent_obj.name, virt_start, virt_end, | |
214 | paddr, perm); | |
215 | ||
5039caf3 EP |
216 | event.type = IOMMU_NOTIFIER_MAP; |
217 | event.entry.target_as = &address_space_memory; | |
218 | event.entry.addr_mask = virt_end - virt_start; | |
219 | event.entry.iova = virt_start; | |
220 | event.entry.perm = perm; | |
221 | event.entry.translated_addr = paddr; | |
15e4c8f0 | 222 | |
5039caf3 | 223 | memory_region_notify_iommu(mr, 0, event); |
15e4c8f0 BB |
224 | } |
225 | ||
226 | static void virtio_iommu_notify_unmap(IOMMUMemoryRegion *mr, hwaddr virt_start, | |
227 | hwaddr virt_end) | |
228 | { | |
5039caf3 | 229 | IOMMUTLBEvent event; |
dde3f08b | 230 | uint64_t delta = virt_end - virt_start; |
15e4c8f0 BB |
231 | |
232 | if (!(mr->iommu_notify_flags & IOMMU_NOTIFIER_UNMAP)) { | |
233 | return; | |
234 | } | |
235 | ||
236 | trace_virtio_iommu_notify_unmap(mr->parent_obj.name, virt_start, virt_end); | |
237 | ||
5039caf3 EP |
238 | event.type = IOMMU_NOTIFIER_UNMAP; |
239 | event.entry.target_as = &address_space_memory; | |
5039caf3 EP |
240 | event.entry.perm = IOMMU_NONE; |
241 | event.entry.translated_addr = 0; | |
dde3f08b EA |
242 | event.entry.addr_mask = delta; |
243 | event.entry.iova = virt_start; | |
15e4c8f0 | 244 | |
dde3f08b EA |
245 | if (delta == UINT64_MAX) { |
246 | memory_region_notify_iommu(mr, 0, event); | |
247 | } | |
248 | ||
249 | ||
250 | while (virt_start != virt_end + 1) { | |
251 | uint64_t mask = dma_aligned_pow2_mask(virt_start, virt_end, 64); | |
252 | ||
253 | event.entry.addr_mask = mask; | |
254 | event.entry.iova = virt_start; | |
255 | memory_region_notify_iommu(mr, 0, event); | |
256 | virt_start += mask + 1; | |
257 | } | |
15e4c8f0 BB |
258 | } |
259 | ||
2f6eeb5f BB |
260 | static gboolean virtio_iommu_notify_unmap_cb(gpointer key, gpointer value, |
261 | gpointer data) | |
262 | { | |
263 | VirtIOIOMMUInterval *interval = (VirtIOIOMMUInterval *) key; | |
264 | IOMMUMemoryRegion *mr = (IOMMUMemoryRegion *) data; | |
265 | ||
266 | virtio_iommu_notify_unmap(mr, interval->low, interval->high); | |
267 | ||
268 | return false; | |
269 | } | |
270 | ||
271 | static gboolean virtio_iommu_notify_map_cb(gpointer key, gpointer value, | |
272 | gpointer data) | |
273 | { | |
274 | VirtIOIOMMUMapping *mapping = (VirtIOIOMMUMapping *) value; | |
275 | VirtIOIOMMUInterval *interval = (VirtIOIOMMUInterval *) key; | |
276 | IOMMUMemoryRegion *mr = (IOMMUMemoryRegion *) data; | |
277 | ||
278 | virtio_iommu_notify_map(mr, interval->low, interval->high, | |
279 | mapping->phys_addr, mapping->flags); | |
280 | ||
281 | return false; | |
282 | } | |
283 | ||
cfb42188 EA |
284 | static void virtio_iommu_detach_endpoint_from_domain(VirtIOIOMMUEndpoint *ep) |
285 | { | |
2f6eeb5f | 286 | VirtIOIOMMUDomain *domain = ep->domain; |
90519b90 | 287 | IOMMUDevice *sdev = container_of(ep->iommu_mr, IOMMUDevice, iommu_mr); |
2f6eeb5f | 288 | |
cfb42188 EA |
289 | if (!ep->domain) { |
290 | return; | |
291 | } | |
2f6eeb5f BB |
292 | g_tree_foreach(domain->mappings, virtio_iommu_notify_unmap_cb, |
293 | ep->iommu_mr); | |
cfb42188 EA |
294 | QLIST_REMOVE(ep, next); |
295 | ep->domain = NULL; | |
90519b90 | 296 | virtio_iommu_switch_address_space(sdev); |
cfb42188 EA |
297 | } |
298 | ||
299 | static VirtIOIOMMUEndpoint *virtio_iommu_get_endpoint(VirtIOIOMMU *s, | |
300 | uint32_t ep_id) | |
301 | { | |
302 | VirtIOIOMMUEndpoint *ep; | |
31aa323f | 303 | IOMMUMemoryRegion *mr; |
cfb42188 EA |
304 | |
305 | ep = g_tree_lookup(s->endpoints, GUINT_TO_POINTER(ep_id)); | |
306 | if (ep) { | |
307 | return ep; | |
308 | } | |
31aa323f JPB |
309 | mr = virtio_iommu_mr(s, ep_id); |
310 | if (!mr) { | |
cfb42188 EA |
311 | return NULL; |
312 | } | |
313 | ep = g_malloc0(sizeof(*ep)); | |
314 | ep->id = ep_id; | |
31aa323f | 315 | ep->iommu_mr = mr; |
cfb42188 EA |
316 | trace_virtio_iommu_get_endpoint(ep_id); |
317 | g_tree_insert(s->endpoints, GUINT_TO_POINTER(ep_id), ep); | |
318 | return ep; | |
319 | } | |
320 | ||
321 | static void virtio_iommu_put_endpoint(gpointer data) | |
322 | { | |
323 | VirtIOIOMMUEndpoint *ep = (VirtIOIOMMUEndpoint *)data; | |
324 | ||
325 | if (ep->domain) { | |
326 | virtio_iommu_detach_endpoint_from_domain(ep); | |
327 | } | |
328 | ||
329 | trace_virtio_iommu_put_endpoint(ep->id); | |
330 | g_free(ep); | |
331 | } | |
332 | ||
333 | static VirtIOIOMMUDomain *virtio_iommu_get_domain(VirtIOIOMMU *s, | |
d9c96f24 JPB |
334 | uint32_t domain_id, |
335 | bool bypass) | |
cfb42188 EA |
336 | { |
337 | VirtIOIOMMUDomain *domain; | |
338 | ||
339 | domain = g_tree_lookup(s->domains, GUINT_TO_POINTER(domain_id)); | |
340 | if (domain) { | |
d9c96f24 JPB |
341 | if (domain->bypass != bypass) { |
342 | return NULL; | |
343 | } | |
cfb42188 EA |
344 | return domain; |
345 | } | |
346 | domain = g_malloc0(sizeof(*domain)); | |
347 | domain->id = domain_id; | |
348 | domain->mappings = g_tree_new_full((GCompareDataFunc)interval_cmp, | |
349 | NULL, (GDestroyNotify)g_free, | |
350 | (GDestroyNotify)g_free); | |
d9c96f24 | 351 | domain->bypass = bypass; |
cfb42188 EA |
352 | g_tree_insert(s->domains, GUINT_TO_POINTER(domain_id), domain); |
353 | QLIST_INIT(&domain->endpoint_list); | |
354 | trace_virtio_iommu_get_domain(domain_id); | |
355 | return domain; | |
356 | } | |
357 | ||
358 | static void virtio_iommu_put_domain(gpointer data) | |
359 | { | |
360 | VirtIOIOMMUDomain *domain = (VirtIOIOMMUDomain *)data; | |
361 | VirtIOIOMMUEndpoint *iter, *tmp; | |
362 | ||
363 | QLIST_FOREACH_SAFE(iter, &domain->endpoint_list, next, tmp) { | |
364 | virtio_iommu_detach_endpoint_from_domain(iter); | |
365 | } | |
366 | g_tree_destroy(domain->mappings); | |
367 | trace_virtio_iommu_put_domain(domain->id); | |
368 | g_free(domain); | |
369 | } | |
370 | ||
371 | static AddressSpace *virtio_iommu_find_add_as(PCIBus *bus, void *opaque, | |
372 | int devfn) | |
373 | { | |
374 | VirtIOIOMMU *s = opaque; | |
375 | IOMMUPciBus *sbus = g_hash_table_lookup(s->as_by_busptr, bus); | |
376 | static uint32_t mr_index; | |
377 | IOMMUDevice *sdev; | |
378 | ||
379 | if (!sbus) { | |
380 | sbus = g_malloc0(sizeof(IOMMUPciBus) + | |
381 | sizeof(IOMMUDevice *) * PCI_DEVFN_MAX); | |
382 | sbus->bus = bus; | |
383 | g_hash_table_insert(s->as_by_busptr, bus, sbus); | |
384 | } | |
385 | ||
386 | sdev = sbus->pbdev[devfn]; | |
387 | if (!sdev) { | |
388 | char *name = g_strdup_printf("%s-%d-%d", | |
389 | TYPE_VIRTIO_IOMMU_MEMORY_REGION, | |
390 | mr_index++, devfn); | |
b21e2380 | 391 | sdev = sbus->pbdev[devfn] = g_new0(IOMMUDevice, 1); |
cfb42188 EA |
392 | |
393 | sdev->viommu = s; | |
394 | sdev->bus = bus; | |
395 | sdev->devfn = devfn; | |
396 | ||
397 | trace_virtio_iommu_init_iommu_mr(name); | |
398 | ||
90519b90 ZD |
399 | memory_region_init(&sdev->root, OBJECT(s), name, UINT64_MAX); |
400 | address_space_init(&sdev->as, &sdev->root, TYPE_VIRTIO_IOMMU); | |
401 | ||
402 | /* | |
403 | * Build the IOMMU disabled container with aliases to the | |
404 | * shared MRs. Note that aliasing to a shared memory region | |
405 | * could help the memory API to detect same FlatViews so we | |
406 | * can have devices to share the same FlatView when in bypass | |
407 | * mode. (either by not configuring virtio-iommu driver or with | |
408 | * "iommu=pt"). It will greatly reduce the total number of | |
409 | * FlatViews of the system hence VM runs faster. | |
410 | */ | |
411 | memory_region_init_alias(&sdev->bypass_mr, OBJECT(s), | |
412 | "system", get_system_memory(), 0, | |
413 | memory_region_size(get_system_memory())); | |
414 | ||
cfb42188 EA |
415 | memory_region_init_iommu(&sdev->iommu_mr, sizeof(sdev->iommu_mr), |
416 | TYPE_VIRTIO_IOMMU_MEMORY_REGION, | |
417 | OBJECT(s), name, | |
418 | UINT64_MAX); | |
90519b90 ZD |
419 | |
420 | /* | |
421 | * Hook both the containers under the root container, we | |
422 | * switch between iommu & bypass MRs by enable/disable | |
423 | * corresponding sub-containers | |
424 | */ | |
425 | memory_region_add_subregion_overlap(&sdev->root, 0, | |
426 | MEMORY_REGION(&sdev->iommu_mr), | |
427 | 0); | |
428 | memory_region_add_subregion_overlap(&sdev->root, 0, | |
429 | &sdev->bypass_mr, 0); | |
430 | ||
431 | virtio_iommu_switch_address_space(sdev); | |
cfb42188 EA |
432 | g_free(name); |
433 | } | |
434 | return &sdev->as; | |
435 | } | |
436 | ||
5442b854 EA |
437 | static int virtio_iommu_attach(VirtIOIOMMU *s, |
438 | struct virtio_iommu_req_attach *req) | |
22c37a10 | 439 | { |
5442b854 EA |
440 | uint32_t domain_id = le32_to_cpu(req->domain); |
441 | uint32_t ep_id = le32_to_cpu(req->endpoint); | |
d9c96f24 | 442 | uint32_t flags = le32_to_cpu(req->flags); |
cfb42188 EA |
443 | VirtIOIOMMUDomain *domain; |
444 | VirtIOIOMMUEndpoint *ep; | |
90519b90 | 445 | IOMMUDevice *sdev; |
5442b854 EA |
446 | |
447 | trace_virtio_iommu_attach(domain_id, ep_id); | |
448 | ||
d9c96f24 JPB |
449 | if (flags & ~VIRTIO_IOMMU_ATTACH_F_BYPASS) { |
450 | return VIRTIO_IOMMU_S_INVAL; | |
451 | } | |
452 | ||
cfb42188 EA |
453 | ep = virtio_iommu_get_endpoint(s, ep_id); |
454 | if (!ep) { | |
455 | return VIRTIO_IOMMU_S_NOENT; | |
456 | } | |
457 | ||
458 | if (ep->domain) { | |
459 | VirtIOIOMMUDomain *previous_domain = ep->domain; | |
460 | /* | |
461 | * the device is already attached to a domain, | |
462 | * detach it first | |
463 | */ | |
464 | virtio_iommu_detach_endpoint_from_domain(ep); | |
465 | if (QLIST_EMPTY(&previous_domain->endpoint_list)) { | |
466 | g_tree_remove(s->domains, GUINT_TO_POINTER(previous_domain->id)); | |
467 | } | |
468 | } | |
469 | ||
d9c96f24 JPB |
470 | domain = virtio_iommu_get_domain(s, domain_id, |
471 | flags & VIRTIO_IOMMU_ATTACH_F_BYPASS); | |
472 | if (!domain) { | |
473 | /* Incompatible bypass flag */ | |
474 | return VIRTIO_IOMMU_S_INVAL; | |
475 | } | |
cfb42188 EA |
476 | QLIST_INSERT_HEAD(&domain->endpoint_list, ep, next); |
477 | ||
478 | ep->domain = domain; | |
90519b90 ZD |
479 | sdev = container_of(ep->iommu_mr, IOMMUDevice, iommu_mr); |
480 | virtio_iommu_switch_address_space(sdev); | |
cfb42188 | 481 | |
2f6eeb5f BB |
482 | /* Replay domain mappings on the associated memory region */ |
483 | g_tree_foreach(domain->mappings, virtio_iommu_notify_map_cb, | |
484 | ep->iommu_mr); | |
485 | ||
cfb42188 | 486 | return VIRTIO_IOMMU_S_OK; |
22c37a10 | 487 | } |
5442b854 EA |
488 | |
489 | static int virtio_iommu_detach(VirtIOIOMMU *s, | |
490 | struct virtio_iommu_req_detach *req) | |
22c37a10 | 491 | { |
5442b854 EA |
492 | uint32_t domain_id = le32_to_cpu(req->domain); |
493 | uint32_t ep_id = le32_to_cpu(req->endpoint); | |
cfb42188 EA |
494 | VirtIOIOMMUDomain *domain; |
495 | VirtIOIOMMUEndpoint *ep; | |
5442b854 EA |
496 | |
497 | trace_virtio_iommu_detach(domain_id, ep_id); | |
498 | ||
cfb42188 EA |
499 | ep = g_tree_lookup(s->endpoints, GUINT_TO_POINTER(ep_id)); |
500 | if (!ep) { | |
501 | return VIRTIO_IOMMU_S_NOENT; | |
502 | } | |
503 | ||
504 | domain = ep->domain; | |
505 | ||
506 | if (!domain || domain->id != domain_id) { | |
507 | return VIRTIO_IOMMU_S_INVAL; | |
508 | } | |
509 | ||
510 | virtio_iommu_detach_endpoint_from_domain(ep); | |
511 | ||
512 | if (QLIST_EMPTY(&domain->endpoint_list)) { | |
513 | g_tree_remove(s->domains, GUINT_TO_POINTER(domain->id)); | |
514 | } | |
515 | return VIRTIO_IOMMU_S_OK; | |
22c37a10 | 516 | } |
5442b854 EA |
517 | |
518 | static int virtio_iommu_map(VirtIOIOMMU *s, | |
519 | struct virtio_iommu_req_map *req) | |
22c37a10 | 520 | { |
5442b854 EA |
521 | uint32_t domain_id = le32_to_cpu(req->domain); |
522 | uint64_t phys_start = le64_to_cpu(req->phys_start); | |
523 | uint64_t virt_start = le64_to_cpu(req->virt_start); | |
524 | uint64_t virt_end = le64_to_cpu(req->virt_end); | |
525 | uint32_t flags = le32_to_cpu(req->flags); | |
fe2cacae EA |
526 | VirtIOIOMMUDomain *domain; |
527 | VirtIOIOMMUInterval *interval; | |
528 | VirtIOIOMMUMapping *mapping; | |
15e4c8f0 | 529 | VirtIOIOMMUEndpoint *ep; |
fe2cacae EA |
530 | |
531 | if (flags & ~VIRTIO_IOMMU_MAP_F_MASK) { | |
532 | return VIRTIO_IOMMU_S_INVAL; | |
533 | } | |
534 | ||
535 | domain = g_tree_lookup(s->domains, GUINT_TO_POINTER(domain_id)); | |
536 | if (!domain) { | |
537 | return VIRTIO_IOMMU_S_NOENT; | |
538 | } | |
539 | ||
d9c96f24 JPB |
540 | if (domain->bypass) { |
541 | return VIRTIO_IOMMU_S_INVAL; | |
542 | } | |
543 | ||
fe2cacae EA |
544 | interval = g_malloc0(sizeof(*interval)); |
545 | ||
546 | interval->low = virt_start; | |
547 | interval->high = virt_end; | |
548 | ||
549 | mapping = g_tree_lookup(domain->mappings, (gpointer)interval); | |
550 | if (mapping) { | |
551 | g_free(interval); | |
552 | return VIRTIO_IOMMU_S_INVAL; | |
553 | } | |
5442b854 EA |
554 | |
555 | trace_virtio_iommu_map(domain_id, virt_start, virt_end, phys_start, flags); | |
556 | ||
fe2cacae EA |
557 | mapping = g_malloc0(sizeof(*mapping)); |
558 | mapping->phys_addr = phys_start; | |
559 | mapping->flags = flags; | |
560 | ||
561 | g_tree_insert(domain->mappings, interval, mapping); | |
562 | ||
15e4c8f0 BB |
563 | QLIST_FOREACH(ep, &domain->endpoint_list, next) { |
564 | virtio_iommu_notify_map(ep->iommu_mr, virt_start, virt_end, phys_start, | |
565 | flags); | |
566 | } | |
567 | ||
fe2cacae | 568 | return VIRTIO_IOMMU_S_OK; |
22c37a10 | 569 | } |
5442b854 EA |
570 | |
571 | static int virtio_iommu_unmap(VirtIOIOMMU *s, | |
572 | struct virtio_iommu_req_unmap *req) | |
22c37a10 | 573 | { |
5442b854 EA |
574 | uint32_t domain_id = le32_to_cpu(req->domain); |
575 | uint64_t virt_start = le64_to_cpu(req->virt_start); | |
576 | uint64_t virt_end = le64_to_cpu(req->virt_end); | |
fe2cacae EA |
577 | VirtIOIOMMUMapping *iter_val; |
578 | VirtIOIOMMUInterval interval, *iter_key; | |
579 | VirtIOIOMMUDomain *domain; | |
15e4c8f0 | 580 | VirtIOIOMMUEndpoint *ep; |
fe2cacae | 581 | int ret = VIRTIO_IOMMU_S_OK; |
5442b854 EA |
582 | |
583 | trace_virtio_iommu_unmap(domain_id, virt_start, virt_end); | |
584 | ||
fe2cacae EA |
585 | domain = g_tree_lookup(s->domains, GUINT_TO_POINTER(domain_id)); |
586 | if (!domain) { | |
587 | return VIRTIO_IOMMU_S_NOENT; | |
588 | } | |
d9c96f24 JPB |
589 | |
590 | if (domain->bypass) { | |
591 | return VIRTIO_IOMMU_S_INVAL; | |
592 | } | |
593 | ||
fe2cacae EA |
594 | interval.low = virt_start; |
595 | interval.high = virt_end; | |
596 | ||
597 | while (g_tree_lookup_extended(domain->mappings, &interval, | |
598 | (void **)&iter_key, (void**)&iter_val)) { | |
599 | uint64_t current_low = iter_key->low; | |
600 | uint64_t current_high = iter_key->high; | |
601 | ||
602 | if (interval.low <= current_low && interval.high >= current_high) { | |
15e4c8f0 BB |
603 | QLIST_FOREACH(ep, &domain->endpoint_list, next) { |
604 | virtio_iommu_notify_unmap(ep->iommu_mr, current_low, | |
605 | current_high); | |
606 | } | |
fe2cacae EA |
607 | g_tree_remove(domain->mappings, iter_key); |
608 | trace_virtio_iommu_unmap_done(domain_id, current_low, current_high); | |
609 | } else { | |
610 | ret = VIRTIO_IOMMU_S_RANGE; | |
611 | break; | |
612 | } | |
613 | } | |
614 | return ret; | |
22c37a10 EA |
615 | } |
616 | ||
1733eebb EA |
617 | static ssize_t virtio_iommu_fill_resv_mem_prop(VirtIOIOMMU *s, uint32_t ep, |
618 | uint8_t *buf, size_t free) | |
619 | { | |
620 | struct virtio_iommu_probe_resv_mem prop = {}; | |
621 | size_t size = sizeof(prop), length = size - sizeof(prop.head), total; | |
622 | int i; | |
623 | ||
624 | total = size * s->nb_reserved_regions; | |
625 | ||
626 | if (total > free) { | |
627 | return -ENOSPC; | |
628 | } | |
629 | ||
630 | for (i = 0; i < s->nb_reserved_regions; i++) { | |
631 | unsigned subtype = s->reserved_regions[i].type; | |
632 | ||
633 | assert(subtype == VIRTIO_IOMMU_RESV_MEM_T_RESERVED || | |
634 | subtype == VIRTIO_IOMMU_RESV_MEM_T_MSI); | |
635 | prop.head.type = cpu_to_le16(VIRTIO_IOMMU_PROBE_T_RESV_MEM); | |
636 | prop.head.length = cpu_to_le16(length); | |
637 | prop.subtype = subtype; | |
638 | prop.start = cpu_to_le64(s->reserved_regions[i].low); | |
639 | prop.end = cpu_to_le64(s->reserved_regions[i].high); | |
640 | ||
641 | memcpy(buf, &prop, size); | |
642 | ||
643 | trace_virtio_iommu_fill_resv_property(ep, prop.subtype, | |
644 | prop.start, prop.end); | |
645 | buf += size; | |
646 | } | |
647 | return total; | |
648 | } | |
649 | ||
650 | /** | |
651 | * virtio_iommu_probe - Fill the probe request buffer with | |
652 | * the properties the device is able to return | |
653 | */ | |
654 | static int virtio_iommu_probe(VirtIOIOMMU *s, | |
655 | struct virtio_iommu_req_probe *req, | |
656 | uint8_t *buf) | |
657 | { | |
658 | uint32_t ep_id = le32_to_cpu(req->endpoint); | |
659 | size_t free = VIOMMU_PROBE_SIZE; | |
660 | ssize_t count; | |
661 | ||
662 | if (!virtio_iommu_mr(s, ep_id)) { | |
663 | return VIRTIO_IOMMU_S_NOENT; | |
664 | } | |
665 | ||
666 | count = virtio_iommu_fill_resv_mem_prop(s, ep_id, buf, free); | |
667 | if (count < 0) { | |
668 | return VIRTIO_IOMMU_S_INVAL; | |
669 | } | |
670 | buf += count; | |
671 | free -= count; | |
672 | ||
673 | return VIRTIO_IOMMU_S_OK; | |
674 | } | |
675 | ||
5442b854 EA |
676 | static int virtio_iommu_iov_to_req(struct iovec *iov, |
677 | unsigned int iov_cnt, | |
678 | void *req, size_t req_sz) | |
679 | { | |
680 | size_t sz, payload_sz = req_sz - sizeof(struct virtio_iommu_req_tail); | |
681 | ||
682 | sz = iov_to_buf(iov, iov_cnt, 0, req, payload_sz); | |
683 | if (unlikely(sz != payload_sz)) { | |
684 | return VIRTIO_IOMMU_S_INVAL; | |
685 | } | |
686 | return 0; | |
687 | } | |
688 | ||
689 | #define virtio_iommu_handle_req(__req) \ | |
690 | static int virtio_iommu_handle_ ## __req(VirtIOIOMMU *s, \ | |
691 | struct iovec *iov, \ | |
692 | unsigned int iov_cnt) \ | |
693 | { \ | |
694 | struct virtio_iommu_req_ ## __req req; \ | |
695 | int ret = virtio_iommu_iov_to_req(iov, iov_cnt, &req, sizeof(req)); \ | |
696 | \ | |
697 | return ret ? ret : virtio_iommu_ ## __req(s, &req); \ | |
698 | } | |
699 | ||
700 | virtio_iommu_handle_req(attach) | |
701 | virtio_iommu_handle_req(detach) | |
702 | virtio_iommu_handle_req(map) | |
703 | virtio_iommu_handle_req(unmap) | |
704 | ||
1733eebb EA |
705 | static int virtio_iommu_handle_probe(VirtIOIOMMU *s, |
706 | struct iovec *iov, | |
707 | unsigned int iov_cnt, | |
708 | uint8_t *buf) | |
709 | { | |
710 | struct virtio_iommu_req_probe req; | |
711 | int ret = virtio_iommu_iov_to_req(iov, iov_cnt, &req, sizeof(req)); | |
712 | ||
713 | return ret ? ret : virtio_iommu_probe(s, &req, buf); | |
714 | } | |
715 | ||
22c37a10 EA |
716 | static void virtio_iommu_handle_command(VirtIODevice *vdev, VirtQueue *vq) |
717 | { | |
718 | VirtIOIOMMU *s = VIRTIO_IOMMU(vdev); | |
719 | struct virtio_iommu_req_head head; | |
720 | struct virtio_iommu_req_tail tail = {}; | |
1733eebb | 721 | size_t output_size = sizeof(tail), sz; |
22c37a10 EA |
722 | VirtQueueElement *elem; |
723 | unsigned int iov_cnt; | |
724 | struct iovec *iov; | |
1733eebb | 725 | void *buf = NULL; |
22c37a10 EA |
726 | |
727 | for (;;) { | |
728 | elem = virtqueue_pop(vq, sizeof(VirtQueueElement)); | |
729 | if (!elem) { | |
730 | return; | |
731 | } | |
732 | ||
733 | if (iov_size(elem->in_sg, elem->in_num) < sizeof(tail) || | |
734 | iov_size(elem->out_sg, elem->out_num) < sizeof(head)) { | |
735 | virtio_error(vdev, "virtio-iommu bad head/tail size"); | |
736 | virtqueue_detach_element(vq, elem, 0); | |
737 | g_free(elem); | |
738 | break; | |
739 | } | |
740 | ||
741 | iov_cnt = elem->out_num; | |
742 | iov = elem->out_sg; | |
743 | sz = iov_to_buf(iov, iov_cnt, 0, &head, sizeof(head)); | |
744 | if (unlikely(sz != sizeof(head))) { | |
745 | tail.status = VIRTIO_IOMMU_S_DEVERR; | |
746 | goto out; | |
747 | } | |
08f2030a | 748 | qemu_rec_mutex_lock(&s->mutex); |
22c37a10 EA |
749 | switch (head.type) { |
750 | case VIRTIO_IOMMU_T_ATTACH: | |
751 | tail.status = virtio_iommu_handle_attach(s, iov, iov_cnt); | |
752 | break; | |
753 | case VIRTIO_IOMMU_T_DETACH: | |
754 | tail.status = virtio_iommu_handle_detach(s, iov, iov_cnt); | |
755 | break; | |
756 | case VIRTIO_IOMMU_T_MAP: | |
757 | tail.status = virtio_iommu_handle_map(s, iov, iov_cnt); | |
758 | break; | |
759 | case VIRTIO_IOMMU_T_UNMAP: | |
760 | tail.status = virtio_iommu_handle_unmap(s, iov, iov_cnt); | |
761 | break; | |
1733eebb EA |
762 | case VIRTIO_IOMMU_T_PROBE: |
763 | { | |
764 | struct virtio_iommu_req_tail *ptail; | |
765 | ||
766 | output_size = s->config.probe_size + sizeof(tail); | |
767 | buf = g_malloc0(output_size); | |
768 | ||
769 | ptail = (struct virtio_iommu_req_tail *) | |
770 | (buf + s->config.probe_size); | |
771 | ptail->status = virtio_iommu_handle_probe(s, iov, iov_cnt, buf); | |
e95e05da | 772 | break; |
1733eebb | 773 | } |
22c37a10 EA |
774 | default: |
775 | tail.status = VIRTIO_IOMMU_S_UNSUPP; | |
776 | } | |
08f2030a | 777 | qemu_rec_mutex_unlock(&s->mutex); |
22c37a10 EA |
778 | |
779 | out: | |
780 | sz = iov_from_buf(elem->in_sg, elem->in_num, 0, | |
1733eebb EA |
781 | buf ? buf : &tail, output_size); |
782 | assert(sz == output_size); | |
22c37a10 | 783 | |
1733eebb | 784 | virtqueue_push(vq, elem, sz); |
22c37a10 EA |
785 | virtio_notify(vdev, vq); |
786 | g_free(elem); | |
1733eebb | 787 | g_free(buf); |
4bf58c72 | 788 | buf = NULL; |
22c37a10 EA |
789 | } |
790 | } | |
791 | ||
a7c1da8a EA |
792 | static void virtio_iommu_report_fault(VirtIOIOMMU *viommu, uint8_t reason, |
793 | int flags, uint32_t endpoint, | |
794 | uint64_t address) | |
795 | { | |
796 | VirtIODevice *vdev = &viommu->parent_obj; | |
797 | VirtQueue *vq = viommu->event_vq; | |
798 | struct virtio_iommu_fault fault; | |
799 | VirtQueueElement *elem; | |
800 | size_t sz; | |
801 | ||
802 | memset(&fault, 0, sizeof(fault)); | |
803 | fault.reason = reason; | |
804 | fault.flags = cpu_to_le32(flags); | |
805 | fault.endpoint = cpu_to_le32(endpoint); | |
806 | fault.address = cpu_to_le64(address); | |
807 | ||
808 | elem = virtqueue_pop(vq, sizeof(VirtQueueElement)); | |
809 | ||
810 | if (!elem) { | |
811 | error_report_once( | |
812 | "no buffer available in event queue to report event"); | |
813 | return; | |
814 | } | |
815 | ||
816 | if (iov_size(elem->in_sg, elem->in_num) < sizeof(fault)) { | |
817 | virtio_error(vdev, "error buffer of wrong size"); | |
818 | virtqueue_detach_element(vq, elem, 0); | |
819 | g_free(elem); | |
820 | return; | |
821 | } | |
822 | ||
823 | sz = iov_from_buf(elem->in_sg, elem->in_num, 0, | |
824 | &fault, sizeof(fault)); | |
825 | assert(sz == sizeof(fault)); | |
826 | ||
827 | trace_virtio_iommu_report_fault(reason, flags, endpoint, address); | |
828 | virtqueue_push(vq, elem, sz); | |
829 | virtio_notify(vdev, vq); | |
830 | g_free(elem); | |
831 | ||
832 | } | |
833 | ||
cfb42188 EA |
834 | static IOMMUTLBEntry virtio_iommu_translate(IOMMUMemoryRegion *mr, hwaddr addr, |
835 | IOMMUAccessFlags flag, | |
836 | int iommu_idx) | |
837 | { | |
838 | IOMMUDevice *sdev = container_of(mr, IOMMUDevice, iommu_mr); | |
ed8449b3 EA |
839 | VirtIOIOMMUInterval interval, *mapping_key; |
840 | VirtIOIOMMUMapping *mapping_value; | |
841 | VirtIOIOMMU *s = sdev->viommu; | |
a7c1da8a | 842 | bool read_fault, write_fault; |
ed8449b3 | 843 | VirtIOIOMMUEndpoint *ep; |
a7c1da8a | 844 | uint32_t sid, flags; |
ed8449b3 | 845 | bool bypass_allowed; |
ed8449b3 | 846 | bool found; |
0f5a3092 | 847 | int i; |
ed8449b3 EA |
848 | |
849 | interval.low = addr; | |
850 | interval.high = addr + 1; | |
cfb42188 EA |
851 | |
852 | IOMMUTLBEntry entry = { | |
853 | .target_as = &address_space_memory, | |
854 | .iova = addr, | |
855 | .translated_addr = addr, | |
ed8449b3 | 856 | .addr_mask = (1 << ctz32(s->config.page_size_mask)) - 1, |
cfb42188 EA |
857 | .perm = IOMMU_NONE, |
858 | }; | |
859 | ||
448179e3 | 860 | bypass_allowed = s->config.bypass; |
ed8449b3 | 861 | |
cfb42188 EA |
862 | sid = virtio_iommu_get_bdf(sdev); |
863 | ||
864 | trace_virtio_iommu_translate(mr->parent_obj.name, sid, addr, flag); | |
08f2030a | 865 | qemu_rec_mutex_lock(&s->mutex); |
ed8449b3 EA |
866 | |
867 | ep = g_tree_lookup(s->endpoints, GUINT_TO_POINTER(sid)); | |
23b5f0ff ZD |
868 | |
869 | if (bypass_allowed) | |
870 | assert(ep && ep->domain && !ep->domain->bypass); | |
871 | ||
ed8449b3 EA |
872 | if (!ep) { |
873 | if (!bypass_allowed) { | |
874 | error_report_once("%s sid=%d is not known!!", __func__, sid); | |
a7c1da8a EA |
875 | virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_UNKNOWN, |
876 | VIRTIO_IOMMU_FAULT_F_ADDRESS, | |
877 | sid, addr); | |
ed8449b3 EA |
878 | } else { |
879 | entry.perm = flag; | |
880 | } | |
881 | goto unlock; | |
882 | } | |
883 | ||
0f5a3092 EA |
884 | for (i = 0; i < s->nb_reserved_regions; i++) { |
885 | ReservedRegion *reg = &s->reserved_regions[i]; | |
886 | ||
887 | if (addr >= reg->low && addr <= reg->high) { | |
888 | switch (reg->type) { | |
889 | case VIRTIO_IOMMU_RESV_MEM_T_MSI: | |
890 | entry.perm = flag; | |
891 | break; | |
892 | case VIRTIO_IOMMU_RESV_MEM_T_RESERVED: | |
893 | default: | |
894 | virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_MAPPING, | |
895 | VIRTIO_IOMMU_FAULT_F_ADDRESS, | |
896 | sid, addr); | |
897 | break; | |
898 | } | |
899 | goto unlock; | |
900 | } | |
901 | } | |
902 | ||
ed8449b3 EA |
903 | if (!ep->domain) { |
904 | if (!bypass_allowed) { | |
905 | error_report_once("%s %02x:%02x.%01x not attached to any domain", | |
906 | __func__, PCI_BUS_NUM(sid), | |
907 | PCI_SLOT(sid), PCI_FUNC(sid)); | |
a7c1da8a EA |
908 | virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_DOMAIN, |
909 | VIRTIO_IOMMU_FAULT_F_ADDRESS, | |
910 | sid, addr); | |
ed8449b3 EA |
911 | } else { |
912 | entry.perm = flag; | |
913 | } | |
914 | goto unlock; | |
d9c96f24 JPB |
915 | } else if (ep->domain->bypass) { |
916 | entry.perm = flag; | |
917 | goto unlock; | |
ed8449b3 EA |
918 | } |
919 | ||
920 | found = g_tree_lookup_extended(ep->domain->mappings, (gpointer)(&interval), | |
921 | (void **)&mapping_key, | |
922 | (void **)&mapping_value); | |
923 | if (!found) { | |
924 | error_report_once("%s no mapping for 0x%"PRIx64" for sid=%d", | |
925 | __func__, addr, sid); | |
a7c1da8a EA |
926 | virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_MAPPING, |
927 | VIRTIO_IOMMU_FAULT_F_ADDRESS, | |
928 | sid, addr); | |
ed8449b3 EA |
929 | goto unlock; |
930 | } | |
931 | ||
a7c1da8a EA |
932 | read_fault = (flag & IOMMU_RO) && |
933 | !(mapping_value->flags & VIRTIO_IOMMU_MAP_F_READ); | |
934 | write_fault = (flag & IOMMU_WO) && | |
935 | !(mapping_value->flags & VIRTIO_IOMMU_MAP_F_WRITE); | |
936 | ||
937 | flags = read_fault ? VIRTIO_IOMMU_FAULT_F_READ : 0; | |
938 | flags |= write_fault ? VIRTIO_IOMMU_FAULT_F_WRITE : 0; | |
939 | if (flags) { | |
ed8449b3 EA |
940 | error_report_once("%s permission error on 0x%"PRIx64"(%d): allowed=%d", |
941 | __func__, addr, flag, mapping_value->flags); | |
a7c1da8a EA |
942 | flags |= VIRTIO_IOMMU_FAULT_F_ADDRESS; |
943 | virtio_iommu_report_fault(s, VIRTIO_IOMMU_FAULT_R_MAPPING, | |
944 | flags | VIRTIO_IOMMU_FAULT_F_ADDRESS, | |
945 | sid, addr); | |
ed8449b3 EA |
946 | goto unlock; |
947 | } | |
948 | entry.translated_addr = addr - mapping_key->low + mapping_value->phys_addr; | |
949 | entry.perm = flag; | |
950 | trace_virtio_iommu_translate_out(addr, entry.translated_addr, sid); | |
951 | ||
952 | unlock: | |
08f2030a | 953 | qemu_rec_mutex_unlock(&s->mutex); |
cfb42188 EA |
954 | return entry; |
955 | } | |
956 | ||
22c37a10 EA |
957 | static void virtio_iommu_get_config(VirtIODevice *vdev, uint8_t *config_data) |
958 | { | |
959 | VirtIOIOMMU *dev = VIRTIO_IOMMU(vdev); | |
3a411b2d EA |
960 | struct virtio_iommu_config *dev_config = &dev->config; |
961 | struct virtio_iommu_config *out_config = (void *)config_data; | |
962 | ||
963 | out_config->page_size_mask = cpu_to_le64(dev_config->page_size_mask); | |
964 | out_config->input_range.start = cpu_to_le64(dev_config->input_range.start); | |
965 | out_config->input_range.end = cpu_to_le64(dev_config->input_range.end); | |
966 | out_config->domain_range.start = cpu_to_le32(dev_config->domain_range.start); | |
967 | out_config->domain_range.end = cpu_to_le32(dev_config->domain_range.end); | |
968 | out_config->probe_size = cpu_to_le32(dev_config->probe_size); | |
448179e3 | 969 | out_config->bypass = dev_config->bypass; |
3a411b2d EA |
970 | |
971 | trace_virtio_iommu_get_config(dev_config->page_size_mask, | |
972 | dev_config->input_range.start, | |
973 | dev_config->input_range.end, | |
974 | dev_config->domain_range.start, | |
975 | dev_config->domain_range.end, | |
448179e3 JPB |
976 | dev_config->probe_size, |
977 | dev_config->bypass); | |
978 | } | |
979 | ||
980 | static void virtio_iommu_set_config(VirtIODevice *vdev, | |
981 | const uint8_t *config_data) | |
982 | { | |
983 | VirtIOIOMMU *dev = VIRTIO_IOMMU(vdev); | |
984 | struct virtio_iommu_config *dev_config = &dev->config; | |
985 | const struct virtio_iommu_config *in_config = (void *)config_data; | |
986 | ||
987 | if (in_config->bypass != dev_config->bypass) { | |
988 | if (!virtio_vdev_has_feature(vdev, VIRTIO_IOMMU_F_BYPASS_CONFIG)) { | |
989 | virtio_error(vdev, "cannot set config.bypass"); | |
990 | return; | |
991 | } else if (in_config->bypass != 0 && in_config->bypass != 1) { | |
992 | virtio_error(vdev, "invalid config.bypass value '%u'", | |
993 | in_config->bypass); | |
994 | return; | |
995 | } | |
996 | dev_config->bypass = in_config->bypass; | |
90519b90 | 997 | virtio_iommu_switch_address_space_all(dev); |
448179e3 JPB |
998 | } |
999 | ||
1000 | trace_virtio_iommu_set_config(in_config->bypass); | |
22c37a10 EA |
1001 | } |
1002 | ||
22c37a10 EA |
1003 | static uint64_t virtio_iommu_get_features(VirtIODevice *vdev, uint64_t f, |
1004 | Error **errp) | |
1005 | { | |
1006 | VirtIOIOMMU *dev = VIRTIO_IOMMU(vdev); | |
1007 | ||
1008 | f |= dev->features; | |
1009 | trace_virtio_iommu_get_features(f); | |
1010 | return f; | |
1011 | } | |
1012 | ||
cfb42188 EA |
1013 | static gint int_cmp(gconstpointer a, gconstpointer b, gpointer user_data) |
1014 | { | |
1015 | guint ua = GPOINTER_TO_UINT(a); | |
1016 | guint ub = GPOINTER_TO_UINT(b); | |
1017 | return (ua > ub) - (ua < ub); | |
1018 | } | |
1019 | ||
308e5e1b BB |
1020 | static gboolean virtio_iommu_remap(gpointer key, gpointer value, gpointer data) |
1021 | { | |
1022 | VirtIOIOMMUMapping *mapping = (VirtIOIOMMUMapping *) value; | |
1023 | VirtIOIOMMUInterval *interval = (VirtIOIOMMUInterval *) key; | |
1024 | IOMMUMemoryRegion *mr = (IOMMUMemoryRegion *) data; | |
1025 | ||
1026 | trace_virtio_iommu_remap(mr->parent_obj.name, interval->low, interval->high, | |
1027 | mapping->phys_addr); | |
1028 | virtio_iommu_notify_map(mr, interval->low, interval->high, | |
1029 | mapping->phys_addr, mapping->flags); | |
1030 | return false; | |
1031 | } | |
1032 | ||
1033 | static void virtio_iommu_replay(IOMMUMemoryRegion *mr, IOMMUNotifier *n) | |
1034 | { | |
1035 | IOMMUDevice *sdev = container_of(mr, IOMMUDevice, iommu_mr); | |
1036 | VirtIOIOMMU *s = sdev->viommu; | |
1037 | uint32_t sid; | |
1038 | VirtIOIOMMUEndpoint *ep; | |
1039 | ||
1040 | sid = virtio_iommu_get_bdf(sdev); | |
1041 | ||
08f2030a | 1042 | qemu_rec_mutex_lock(&s->mutex); |
308e5e1b BB |
1043 | |
1044 | if (!s->endpoints) { | |
1045 | goto unlock; | |
1046 | } | |
1047 | ||
1048 | ep = g_tree_lookup(s->endpoints, GUINT_TO_POINTER(sid)); | |
1049 | if (!ep || !ep->domain) { | |
1050 | goto unlock; | |
1051 | } | |
1052 | ||
1053 | g_tree_foreach(ep->domain->mappings, virtio_iommu_remap, mr); | |
1054 | ||
1055 | unlock: | |
08f2030a | 1056 | qemu_rec_mutex_unlock(&s->mutex); |
308e5e1b BB |
1057 | } |
1058 | ||
6978bfaa BB |
1059 | static int virtio_iommu_notify_flag_changed(IOMMUMemoryRegion *iommu_mr, |
1060 | IOMMUNotifierFlag old, | |
1061 | IOMMUNotifierFlag new, | |
1062 | Error **errp) | |
1063 | { | |
958ec334 PX |
1064 | if (new & IOMMU_NOTIFIER_DEVIOTLB_UNMAP) { |
1065 | error_setg(errp, "Virtio-iommu does not support dev-iotlb yet"); | |
1066 | return -EINVAL; | |
1067 | } | |
1068 | ||
6978bfaa BB |
1069 | if (old == IOMMU_NOTIFIER_NONE) { |
1070 | trace_virtio_iommu_notify_flag_add(iommu_mr->parent_obj.name); | |
1071 | } else if (new == IOMMU_NOTIFIER_NONE) { | |
1072 | trace_virtio_iommu_notify_flag_del(iommu_mr->parent_obj.name); | |
1073 | } | |
1074 | return 0; | |
1075 | } | |
1076 | ||
5c3cfe33 BB |
1077 | /* |
1078 | * The default mask (TARGET_PAGE_MASK) is the smallest supported guest granule, | |
1079 | * for example 0xfffffffffffff000. When an assigned device has page size | |
1080 | * restrictions due to the hardware IOMMU configuration, apply this restriction | |
1081 | * to the mask. | |
1082 | */ | |
1083 | static int virtio_iommu_set_page_size_mask(IOMMUMemoryRegion *mr, | |
1084 | uint64_t new_mask, | |
1085 | Error **errp) | |
1086 | { | |
1087 | IOMMUDevice *sdev = container_of(mr, IOMMUDevice, iommu_mr); | |
1088 | VirtIOIOMMU *s = sdev->viommu; | |
1089 | uint64_t cur_mask = s->config.page_size_mask; | |
1090 | ||
1091 | trace_virtio_iommu_set_page_size_mask(mr->parent_obj.name, cur_mask, | |
1092 | new_mask); | |
1093 | ||
1094 | if ((cur_mask & new_mask) == 0) { | |
1095 | error_setg(errp, "virtio-iommu page mask 0x%"PRIx64 | |
1096 | " is incompatible with mask 0x%"PRIx64, cur_mask, new_mask); | |
1097 | return -1; | |
1098 | } | |
1099 | ||
1100 | /* | |
1101 | * After the machine is finalized, we can't change the mask anymore. If by | |
1102 | * chance the hotplugged device supports the same granule, we can still | |
1103 | * accept it. Having a different masks is possible but the guest will use | |
1104 | * sub-optimal block sizes, so warn about it. | |
1105 | */ | |
2f181fbd | 1106 | if (phase_check(PHASE_MACHINE_READY)) { |
5c3cfe33 BB |
1107 | int new_granule = ctz64(new_mask); |
1108 | int cur_granule = ctz64(cur_mask); | |
1109 | ||
1110 | if (new_granule != cur_granule) { | |
1111 | error_setg(errp, "virtio-iommu page mask 0x%"PRIx64 | |
1112 | " is incompatible with mask 0x%"PRIx64, cur_mask, | |
1113 | new_mask); | |
1114 | return -1; | |
1115 | } else if (new_mask != cur_mask) { | |
1116 | warn_report("virtio-iommu page mask 0x%"PRIx64 | |
1117 | " does not match 0x%"PRIx64, cur_mask, new_mask); | |
1118 | } | |
1119 | return 0; | |
1120 | } | |
1121 | ||
1122 | s->config.page_size_mask &= new_mask; | |
1123 | return 0; | |
1124 | } | |
1125 | ||
448179e3 JPB |
1126 | static void virtio_iommu_system_reset(void *opaque) |
1127 | { | |
1128 | VirtIOIOMMU *s = opaque; | |
1129 | ||
1130 | trace_virtio_iommu_system_reset(); | |
1131 | ||
1132 | /* | |
1133 | * config.bypass is sticky across device reset, but should be restored on | |
1134 | * system reset | |
1135 | */ | |
1136 | s->config.bypass = s->boot_bypass; | |
90519b90 ZD |
1137 | virtio_iommu_switch_address_space_all(s); |
1138 | ||
448179e3 JPB |
1139 | } |
1140 | ||
22c37a10 EA |
1141 | static void virtio_iommu_device_realize(DeviceState *dev, Error **errp) |
1142 | { | |
1143 | VirtIODevice *vdev = VIRTIO_DEVICE(dev); | |
1144 | VirtIOIOMMU *s = VIRTIO_IOMMU(dev); | |
1145 | ||
3857cd5c | 1146 | virtio_init(vdev, VIRTIO_ID_IOMMU, sizeof(struct virtio_iommu_config)); |
22c37a10 | 1147 | |
cfb42188 EA |
1148 | memset(s->iommu_pcibus_by_bus_num, 0, sizeof(s->iommu_pcibus_by_bus_num)); |
1149 | ||
22c37a10 EA |
1150 | s->req_vq = virtio_add_queue(vdev, VIOMMU_DEFAULT_QUEUE_SIZE, |
1151 | virtio_iommu_handle_command); | |
1152 | s->event_vq = virtio_add_queue(vdev, VIOMMU_DEFAULT_QUEUE_SIZE, NULL); | |
1153 | ||
90519b90 ZD |
1154 | /* |
1155 | * config.bypass is needed to get initial address space early, such as | |
1156 | * in vfio realize | |
1157 | */ | |
1158 | s->config.bypass = s->boot_bypass; | |
22c37a10 | 1159 | s->config.page_size_mask = TARGET_PAGE_MASK; |
6b77ae05 EA |
1160 | s->config.input_range.end = UINT64_MAX; |
1161 | s->config.domain_range.end = UINT32_MAX; | |
1733eebb | 1162 | s->config.probe_size = VIOMMU_PROBE_SIZE; |
22c37a10 EA |
1163 | |
1164 | virtio_add_feature(&s->features, VIRTIO_RING_F_EVENT_IDX); | |
1165 | virtio_add_feature(&s->features, VIRTIO_RING_F_INDIRECT_DESC); | |
1166 | virtio_add_feature(&s->features, VIRTIO_F_VERSION_1); | |
1167 | virtio_add_feature(&s->features, VIRTIO_IOMMU_F_INPUT_RANGE); | |
1168 | virtio_add_feature(&s->features, VIRTIO_IOMMU_F_DOMAIN_RANGE); | |
1169 | virtio_add_feature(&s->features, VIRTIO_IOMMU_F_MAP_UNMAP); | |
22c37a10 | 1170 | virtio_add_feature(&s->features, VIRTIO_IOMMU_F_MMIO); |
1733eebb | 1171 | virtio_add_feature(&s->features, VIRTIO_IOMMU_F_PROBE); |
448179e3 | 1172 | virtio_add_feature(&s->features, VIRTIO_IOMMU_F_BYPASS_CONFIG); |
22c37a10 | 1173 | |
08f2030a | 1174 | qemu_rec_mutex_init(&s->mutex); |
cfb42188 EA |
1175 | |
1176 | s->as_by_busptr = g_hash_table_new_full(NULL, NULL, NULL, g_free); | |
1177 | ||
1178 | if (s->primary_bus) { | |
1179 | pci_setup_iommu(s->primary_bus, virtio_iommu_find_add_as, s); | |
1180 | } else { | |
1181 | error_setg(errp, "VIRTIO-IOMMU is not attached to any PCI bus!"); | |
1182 | } | |
448179e3 JPB |
1183 | |
1184 | qemu_register_reset(virtio_iommu_system_reset, s); | |
22c37a10 EA |
1185 | } |
1186 | ||
b69c3c21 | 1187 | static void virtio_iommu_device_unrealize(DeviceState *dev) |
22c37a10 EA |
1188 | { |
1189 | VirtIODevice *vdev = VIRTIO_DEVICE(dev); | |
cfb42188 EA |
1190 | VirtIOIOMMU *s = VIRTIO_IOMMU(dev); |
1191 | ||
448179e3 JPB |
1192 | qemu_unregister_reset(virtio_iommu_system_reset, s); |
1193 | ||
de38ed30 | 1194 | g_hash_table_destroy(s->as_by_busptr); |
59bf980d EA |
1195 | if (s->domains) { |
1196 | g_tree_destroy(s->domains); | |
1197 | } | |
1198 | if (s->endpoints) { | |
1199 | g_tree_destroy(s->endpoints); | |
1200 | } | |
22c37a10 | 1201 | |
08f2030a ZD |
1202 | qemu_rec_mutex_destroy(&s->mutex); |
1203 | ||
de38ed30 PN |
1204 | virtio_delete_queue(s->req_vq); |
1205 | virtio_delete_queue(s->event_vq); | |
22c37a10 EA |
1206 | virtio_cleanup(vdev); |
1207 | } | |
1208 | ||
1209 | static void virtio_iommu_device_reset(VirtIODevice *vdev) | |
1210 | { | |
cfb42188 EA |
1211 | VirtIOIOMMU *s = VIRTIO_IOMMU(vdev); |
1212 | ||
22c37a10 | 1213 | trace_virtio_iommu_device_reset(); |
cfb42188 EA |
1214 | |
1215 | if (s->domains) { | |
1216 | g_tree_destroy(s->domains); | |
1217 | } | |
1218 | if (s->endpoints) { | |
1219 | g_tree_destroy(s->endpoints); | |
1220 | } | |
1221 | s->domains = g_tree_new_full((GCompareDataFunc)int_cmp, | |
1222 | NULL, NULL, virtio_iommu_put_domain); | |
1223 | s->endpoints = g_tree_new_full((GCompareDataFunc)int_cmp, | |
1224 | NULL, NULL, virtio_iommu_put_endpoint); | |
22c37a10 EA |
1225 | } |
1226 | ||
1227 | static void virtio_iommu_set_status(VirtIODevice *vdev, uint8_t status) | |
1228 | { | |
1229 | trace_virtio_iommu_device_status(status); | |
1230 | } | |
1231 | ||
1232 | static void virtio_iommu_instance_init(Object *obj) | |
1233 | { | |
1234 | } | |
1235 | ||
bd0ab870 EA |
1236 | #define VMSTATE_INTERVAL \ |
1237 | { \ | |
1238 | .name = "interval", \ | |
1239 | .version_id = 1, \ | |
1240 | .minimum_version_id = 1, \ | |
1241 | .fields = (VMStateField[]) { \ | |
1242 | VMSTATE_UINT64(low, VirtIOIOMMUInterval), \ | |
1243 | VMSTATE_UINT64(high, VirtIOIOMMUInterval), \ | |
1244 | VMSTATE_END_OF_LIST() \ | |
1245 | } \ | |
1246 | } | |
1247 | ||
1248 | #define VMSTATE_MAPPING \ | |
1249 | { \ | |
1250 | .name = "mapping", \ | |
1251 | .version_id = 1, \ | |
1252 | .minimum_version_id = 1, \ | |
1253 | .fields = (VMStateField[]) { \ | |
1254 | VMSTATE_UINT64(phys_addr, VirtIOIOMMUMapping),\ | |
1255 | VMSTATE_UINT32(flags, VirtIOIOMMUMapping), \ | |
1256 | VMSTATE_END_OF_LIST() \ | |
1257 | }, \ | |
1258 | } | |
1259 | ||
1260 | static const VMStateDescription vmstate_interval_mapping[2] = { | |
1261 | VMSTATE_MAPPING, /* value */ | |
1262 | VMSTATE_INTERVAL /* key */ | |
1263 | }; | |
1264 | ||
1265 | static int domain_preload(void *opaque) | |
1266 | { | |
1267 | VirtIOIOMMUDomain *domain = opaque; | |
1268 | ||
1269 | domain->mappings = g_tree_new_full((GCompareDataFunc)interval_cmp, | |
1270 | NULL, g_free, g_free); | |
1271 | return 0; | |
1272 | } | |
1273 | ||
1274 | static const VMStateDescription vmstate_endpoint = { | |
1275 | .name = "endpoint", | |
1276 | .version_id = 1, | |
1277 | .minimum_version_id = 1, | |
1278 | .fields = (VMStateField[]) { | |
1279 | VMSTATE_UINT32(id, VirtIOIOMMUEndpoint), | |
1280 | VMSTATE_END_OF_LIST() | |
1281 | } | |
1282 | }; | |
1283 | ||
1284 | static const VMStateDescription vmstate_domain = { | |
1285 | .name = "domain", | |
d9c96f24 JPB |
1286 | .version_id = 2, |
1287 | .minimum_version_id = 2, | |
bd0ab870 EA |
1288 | .pre_load = domain_preload, |
1289 | .fields = (VMStateField[]) { | |
1290 | VMSTATE_UINT32(id, VirtIOIOMMUDomain), | |
1291 | VMSTATE_GTREE_V(mappings, VirtIOIOMMUDomain, 1, | |
1292 | vmstate_interval_mapping, | |
1293 | VirtIOIOMMUInterval, VirtIOIOMMUMapping), | |
1294 | VMSTATE_QLIST_V(endpoint_list, VirtIOIOMMUDomain, 1, | |
1295 | vmstate_endpoint, VirtIOIOMMUEndpoint, next), | |
d9c96f24 | 1296 | VMSTATE_BOOL_V(bypass, VirtIOIOMMUDomain, 2), |
bd0ab870 EA |
1297 | VMSTATE_END_OF_LIST() |
1298 | } | |
1299 | }; | |
1300 | ||
1301 | static gboolean reconstruct_endpoints(gpointer key, gpointer value, | |
1302 | gpointer data) | |
1303 | { | |
1304 | VirtIOIOMMU *s = (VirtIOIOMMU *)data; | |
1305 | VirtIOIOMMUDomain *d = (VirtIOIOMMUDomain *)value; | |
1306 | VirtIOIOMMUEndpoint *iter; | |
31aa323f | 1307 | IOMMUMemoryRegion *mr; |
bd0ab870 EA |
1308 | |
1309 | QLIST_FOREACH(iter, &d->endpoint_list, next) { | |
31aa323f JPB |
1310 | mr = virtio_iommu_mr(s, iter->id); |
1311 | assert(mr); | |
1312 | ||
bd0ab870 | 1313 | iter->domain = d; |
31aa323f | 1314 | iter->iommu_mr = mr; |
bd0ab870 EA |
1315 | g_tree_insert(s->endpoints, GUINT_TO_POINTER(iter->id), iter); |
1316 | } | |
1317 | return false; /* continue the domain traversal */ | |
1318 | } | |
1319 | ||
1320 | static int iommu_post_load(void *opaque, int version_id) | |
1321 | { | |
1322 | VirtIOIOMMU *s = opaque; | |
1323 | ||
1324 | g_tree_foreach(s->domains, reconstruct_endpoints, s); | |
1325 | return 0; | |
1326 | } | |
1327 | ||
1328 | static const VMStateDescription vmstate_virtio_iommu_device = { | |
1329 | .name = "virtio-iommu-device", | |
448179e3 JPB |
1330 | .minimum_version_id = 2, |
1331 | .version_id = 2, | |
bd0ab870 EA |
1332 | .post_load = iommu_post_load, |
1333 | .fields = (VMStateField[]) { | |
d9c96f24 | 1334 | VMSTATE_GTREE_DIRECT_KEY_V(domains, VirtIOIOMMU, 2, |
bd0ab870 | 1335 | &vmstate_domain, VirtIOIOMMUDomain), |
448179e3 | 1336 | VMSTATE_UINT8_V(config.bypass, VirtIOIOMMU, 2), |
bd0ab870 EA |
1337 | VMSTATE_END_OF_LIST() |
1338 | }, | |
1339 | }; | |
1340 | ||
22c37a10 EA |
1341 | static const VMStateDescription vmstate_virtio_iommu = { |
1342 | .name = "virtio-iommu", | |
448179e3 | 1343 | .minimum_version_id = 2, |
bd0ab870 | 1344 | .priority = MIG_PRI_IOMMU, |
448179e3 | 1345 | .version_id = 2, |
22c37a10 EA |
1346 | .fields = (VMStateField[]) { |
1347 | VMSTATE_VIRTIO_DEVICE, | |
1348 | VMSTATE_END_OF_LIST() | |
1349 | }, | |
1350 | }; | |
1351 | ||
1352 | static Property virtio_iommu_properties[] = { | |
1353 | DEFINE_PROP_LINK("primary-bus", VirtIOIOMMU, primary_bus, "PCI", PCIBus *), | |
448179e3 | 1354 | DEFINE_PROP_BOOL("boot-bypass", VirtIOIOMMU, boot_bypass, true), |
22c37a10 EA |
1355 | DEFINE_PROP_END_OF_LIST(), |
1356 | }; | |
1357 | ||
1358 | static void virtio_iommu_class_init(ObjectClass *klass, void *data) | |
1359 | { | |
1360 | DeviceClass *dc = DEVICE_CLASS(klass); | |
1361 | VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass); | |
1362 | ||
1363 | device_class_set_props(dc, virtio_iommu_properties); | |
1364 | dc->vmsd = &vmstate_virtio_iommu; | |
1365 | ||
1366 | set_bit(DEVICE_CATEGORY_MISC, dc->categories); | |
1367 | vdc->realize = virtio_iommu_device_realize; | |
1368 | vdc->unrealize = virtio_iommu_device_unrealize; | |
1369 | vdc->reset = virtio_iommu_device_reset; | |
1370 | vdc->get_config = virtio_iommu_get_config; | |
448179e3 | 1371 | vdc->set_config = virtio_iommu_set_config; |
22c37a10 EA |
1372 | vdc->get_features = virtio_iommu_get_features; |
1373 | vdc->set_status = virtio_iommu_set_status; | |
1374 | vdc->vmsd = &vmstate_virtio_iommu_device; | |
1375 | } | |
1376 | ||
cfb42188 EA |
1377 | static void virtio_iommu_memory_region_class_init(ObjectClass *klass, |
1378 | void *data) | |
1379 | { | |
1380 | IOMMUMemoryRegionClass *imrc = IOMMU_MEMORY_REGION_CLASS(klass); | |
1381 | ||
1382 | imrc->translate = virtio_iommu_translate; | |
308e5e1b | 1383 | imrc->replay = virtio_iommu_replay; |
6978bfaa | 1384 | imrc->notify_flag_changed = virtio_iommu_notify_flag_changed; |
5c3cfe33 | 1385 | imrc->iommu_set_page_size_mask = virtio_iommu_set_page_size_mask; |
cfb42188 EA |
1386 | } |
1387 | ||
22c37a10 EA |
1388 | static const TypeInfo virtio_iommu_info = { |
1389 | .name = TYPE_VIRTIO_IOMMU, | |
1390 | .parent = TYPE_VIRTIO_DEVICE, | |
1391 | .instance_size = sizeof(VirtIOIOMMU), | |
1392 | .instance_init = virtio_iommu_instance_init, | |
1393 | .class_init = virtio_iommu_class_init, | |
1394 | }; | |
1395 | ||
cfb42188 EA |
1396 | static const TypeInfo virtio_iommu_memory_region_info = { |
1397 | .parent = TYPE_IOMMU_MEMORY_REGION, | |
1398 | .name = TYPE_VIRTIO_IOMMU_MEMORY_REGION, | |
1399 | .class_init = virtio_iommu_memory_region_class_init, | |
1400 | }; | |
1401 | ||
22c37a10 EA |
1402 | static void virtio_register_types(void) |
1403 | { | |
1404 | type_register_static(&virtio_iommu_info); | |
cfb42188 | 1405 | type_register_static(&virtio_iommu_memory_region_info); |
22c37a10 EA |
1406 | } |
1407 | ||
1408 | type_init(virtio_register_types) |