]>
Commit | Line | Data |
---|---|---|
53c25cea PB |
1 | /* |
2 | * Virtio PCI Bindings | |
3 | * | |
4 | * Copyright IBM, Corp. 2007 | |
5 | * Copyright (c) 2009 CodeSourcery | |
6 | * | |
7 | * Authors: | |
8 | * Anthony Liguori <aliguori@us.ibm.com> | |
9 | * Paul Brook <paul@codesourcery.com> | |
10 | * | |
11 | * This work is licensed under the terms of the GNU GPL, version 2. See | |
12 | * the COPYING file in the top-level directory. | |
13 | * | |
6b620ca3 PB |
14 | * Contributions after 2012-01-13 are licensed under the terms of the |
15 | * GNU GPL, version 2 or (at your option) any later version. | |
53c25cea PB |
16 | */ |
17 | ||
18 | #include <inttypes.h> | |
19 | ||
20 | #include "virtio.h" | |
8172539d MT |
21 | #include "virtio-blk.h" |
22 | #include "virtio-net.h" | |
6b331efb | 23 | #include "virtio-serial.h" |
53c25cea | 24 | #include "pci.h" |
2f792016 | 25 | #include "qemu-error.h" |
aba800a3 | 26 | #include "msix.h" |
a1e0fea5 | 27 | #include "net.h" |
97b15621 | 28 | #include "loader.h" |
ade80dc8 | 29 | #include "kvm.h" |
2446333c | 30 | #include "blockdev.h" |
9fe1ebeb | 31 | #include "virtio-pci.h" |
1129714f | 32 | #include "range.h" |
53c25cea PB |
33 | |
34 | /* from Linux's linux/virtio_pci.h */ | |
35 | ||
36 | /* A 32-bit r/o bitmask of the features supported by the host */ | |
37 | #define VIRTIO_PCI_HOST_FEATURES 0 | |
38 | ||
39 | /* A 32-bit r/w bitmask of features activated by the guest */ | |
40 | #define VIRTIO_PCI_GUEST_FEATURES 4 | |
41 | ||
42 | /* A 32-bit r/w PFN for the currently selected queue */ | |
43 | #define VIRTIO_PCI_QUEUE_PFN 8 | |
44 | ||
45 | /* A 16-bit r/o queue size for the currently selected queue */ | |
46 | #define VIRTIO_PCI_QUEUE_NUM 12 | |
47 | ||
48 | /* A 16-bit r/w queue selector */ | |
49 | #define VIRTIO_PCI_QUEUE_SEL 14 | |
50 | ||
51 | /* A 16-bit r/w queue notifier */ | |
52 | #define VIRTIO_PCI_QUEUE_NOTIFY 16 | |
53 | ||
54 | /* An 8-bit device status register. */ | |
55 | #define VIRTIO_PCI_STATUS 18 | |
56 | ||
57 | /* An 8-bit r/o interrupt status register. Reading the value will return the | |
58 | * current contents of the ISR and will also clear it. This is effectively | |
59 | * a read-and-acknowledge. */ | |
60 | #define VIRTIO_PCI_ISR 19 | |
61 | ||
aba800a3 MT |
62 | /* MSI-X registers: only enabled if MSI-X is enabled. */ |
63 | /* A 16-bit vector for configuration changes. */ | |
64 | #define VIRTIO_MSI_CONFIG_VECTOR 20 | |
65 | /* A 16-bit vector for selected queue notifications. */ | |
66 | #define VIRTIO_MSI_QUEUE_VECTOR 22 | |
67 | ||
68 | /* Config space size */ | |
69 | #define VIRTIO_PCI_CONFIG_NOMSI 20 | |
70 | #define VIRTIO_PCI_CONFIG_MSI 24 | |
71 | #define VIRTIO_PCI_REGION_SIZE(dev) (msix_present(dev) ? \ | |
72 | VIRTIO_PCI_CONFIG_MSI : \ | |
73 | VIRTIO_PCI_CONFIG_NOMSI) | |
74 | ||
75 | /* The remaining space is defined by each driver as the per-driver | |
76 | * configuration space */ | |
77 | #define VIRTIO_PCI_CONFIG(dev) (msix_enabled(dev) ? \ | |
78 | VIRTIO_PCI_CONFIG_MSI : \ | |
79 | VIRTIO_PCI_CONFIG_NOMSI) | |
53c25cea | 80 | |
53c25cea PB |
81 | /* How many bits to shift physical queue address written to QUEUE_PFN. |
82 | * 12 is historical, and due to x86 page size. */ | |
83 | #define VIRTIO_PCI_QUEUE_ADDR_SHIFT 12 | |
84 | ||
3dbca8e6 SH |
85 | /* Flags track per-device state like workarounds for quirks in older guests. */ |
86 | #define VIRTIO_PCI_FLAG_BUS_MASTER_BUG (1 << 0) | |
c81131db | 87 | |
53c25cea PB |
88 | /* QEMU doesn't strictly need write barriers since everything runs in |
89 | * lock-step. We'll leave the calls to wmb() in though to make it obvious for | |
90 | * KVM or if kqemu gets SMP support. | |
91 | */ | |
92 | #define wmb() do { } while (0) | |
93 | ||
53c25cea PB |
94 | /* virtio device */ |
95 | ||
7055e687 | 96 | static void virtio_pci_notify(void *opaque, uint16_t vector) |
53c25cea PB |
97 | { |
98 | VirtIOPCIProxy *proxy = opaque; | |
aba800a3 MT |
99 | if (msix_enabled(&proxy->pci_dev)) |
100 | msix_notify(&proxy->pci_dev, vector); | |
101 | else | |
102 | qemu_set_irq(proxy->pci_dev.irq[0], proxy->vdev->isr & 1); | |
53c25cea PB |
103 | } |
104 | ||
ff24bd58 MT |
105 | static void virtio_pci_save_config(void * opaque, QEMUFile *f) |
106 | { | |
107 | VirtIOPCIProxy *proxy = opaque; | |
108 | pci_device_save(&proxy->pci_dev, f); | |
109 | msix_save(&proxy->pci_dev, f); | |
110 | if (msix_present(&proxy->pci_dev)) | |
111 | qemu_put_be16(f, proxy->vdev->config_vector); | |
112 | } | |
113 | ||
114 | static void virtio_pci_save_queue(void * opaque, int n, QEMUFile *f) | |
115 | { | |
116 | VirtIOPCIProxy *proxy = opaque; | |
117 | if (msix_present(&proxy->pci_dev)) | |
118 | qemu_put_be16(f, virtio_queue_vector(proxy->vdev, n)); | |
119 | } | |
120 | ||
121 | static int virtio_pci_load_config(void * opaque, QEMUFile *f) | |
122 | { | |
123 | VirtIOPCIProxy *proxy = opaque; | |
124 | int ret; | |
125 | ret = pci_device_load(&proxy->pci_dev, f); | |
e6da7680 | 126 | if (ret) { |
ff24bd58 | 127 | return ret; |
e6da7680 | 128 | } |
ff24bd58 | 129 | msix_load(&proxy->pci_dev, f); |
e6da7680 | 130 | if (msix_present(&proxy->pci_dev)) { |
ff24bd58 | 131 | qemu_get_be16s(f, &proxy->vdev->config_vector); |
e6da7680 MT |
132 | } else { |
133 | proxy->vdev->config_vector = VIRTIO_NO_VECTOR; | |
134 | } | |
135 | if (proxy->vdev->config_vector != VIRTIO_NO_VECTOR) { | |
136 | return msix_vector_use(&proxy->pci_dev, proxy->vdev->config_vector); | |
137 | } | |
ff24bd58 MT |
138 | return 0; |
139 | } | |
140 | ||
141 | static int virtio_pci_load_queue(void * opaque, int n, QEMUFile *f) | |
142 | { | |
143 | VirtIOPCIProxy *proxy = opaque; | |
144 | uint16_t vector; | |
e6da7680 MT |
145 | if (msix_present(&proxy->pci_dev)) { |
146 | qemu_get_be16s(f, &vector); | |
147 | } else { | |
148 | vector = VIRTIO_NO_VECTOR; | |
149 | } | |
ff24bd58 | 150 | virtio_queue_set_vector(proxy->vdev, n, vector); |
e6da7680 MT |
151 | if (vector != VIRTIO_NO_VECTOR) { |
152 | return msix_vector_use(&proxy->pci_dev, vector); | |
153 | } | |
ff24bd58 MT |
154 | return 0; |
155 | } | |
156 | ||
25db9ebe SH |
157 | static int virtio_pci_set_host_notifier_internal(VirtIOPCIProxy *proxy, |
158 | int n, bool assign) | |
159 | { | |
160 | VirtQueue *vq = virtio_get_queue(proxy->vdev, n); | |
161 | EventNotifier *notifier = virtio_queue_get_host_notifier(vq); | |
da146d0a AK |
162 | int r = 0; |
163 | ||
25db9ebe SH |
164 | if (assign) { |
165 | r = event_notifier_init(notifier, 1); | |
166 | if (r < 0) { | |
b36e3914 MT |
167 | error_report("%s: unable to init event notifier: %d", |
168 | __func__, r); | |
25db9ebe SH |
169 | return r; |
170 | } | |
da146d0a AK |
171 | memory_region_add_eventfd(&proxy->bar, VIRTIO_PCI_QUEUE_NOTIFY, 2, |
172 | true, n, event_notifier_get_fd(notifier)); | |
25db9ebe | 173 | } else { |
da146d0a AK |
174 | memory_region_del_eventfd(&proxy->bar, VIRTIO_PCI_QUEUE_NOTIFY, 2, |
175 | true, n, event_notifier_get_fd(notifier)); | |
25db9ebe SH |
176 | /* Handle the race condition where the guest kicked and we deassigned |
177 | * before we got around to handling the kick. | |
178 | */ | |
179 | if (event_notifier_test_and_clear(notifier)) { | |
180 | virtio_queue_notify_vq(vq); | |
181 | } | |
182 | ||
183 | event_notifier_cleanup(notifier); | |
184 | } | |
185 | return r; | |
186 | } | |
187 | ||
188 | static void virtio_pci_host_notifier_read(void *opaque) | |
189 | { | |
190 | VirtQueue *vq = opaque; | |
191 | EventNotifier *n = virtio_queue_get_host_notifier(vq); | |
192 | if (event_notifier_test_and_clear(n)) { | |
193 | virtio_queue_notify_vq(vq); | |
194 | } | |
195 | } | |
196 | ||
197 | static void virtio_pci_set_host_notifier_fd_handler(VirtIOPCIProxy *proxy, | |
198 | int n, bool assign) | |
199 | { | |
200 | VirtQueue *vq = virtio_get_queue(proxy->vdev, n); | |
201 | EventNotifier *notifier = virtio_queue_get_host_notifier(vq); | |
202 | if (assign) { | |
203 | qemu_set_fd_handler(event_notifier_get_fd(notifier), | |
204 | virtio_pci_host_notifier_read, NULL, vq); | |
205 | } else { | |
206 | qemu_set_fd_handler(event_notifier_get_fd(notifier), | |
207 | NULL, NULL, NULL); | |
208 | } | |
209 | } | |
210 | ||
b36e3914 | 211 | static void virtio_pci_start_ioeventfd(VirtIOPCIProxy *proxy) |
25db9ebe SH |
212 | { |
213 | int n, r; | |
214 | ||
215 | if (!(proxy->flags & VIRTIO_PCI_FLAG_USE_IOEVENTFD) || | |
216 | proxy->ioeventfd_disabled || | |
217 | proxy->ioeventfd_started) { | |
b36e3914 | 218 | return; |
25db9ebe SH |
219 | } |
220 | ||
221 | for (n = 0; n < VIRTIO_PCI_QUEUE_MAX; n++) { | |
222 | if (!virtio_queue_get_num(proxy->vdev, n)) { | |
223 | continue; | |
224 | } | |
225 | ||
226 | r = virtio_pci_set_host_notifier_internal(proxy, n, true); | |
227 | if (r < 0) { | |
228 | goto assign_error; | |
229 | } | |
230 | ||
231 | virtio_pci_set_host_notifier_fd_handler(proxy, n, true); | |
232 | } | |
233 | proxy->ioeventfd_started = true; | |
b36e3914 | 234 | return; |
25db9ebe SH |
235 | |
236 | assign_error: | |
237 | while (--n >= 0) { | |
238 | if (!virtio_queue_get_num(proxy->vdev, n)) { | |
239 | continue; | |
240 | } | |
241 | ||
242 | virtio_pci_set_host_notifier_fd_handler(proxy, n, false); | |
b36e3914 MT |
243 | r = virtio_pci_set_host_notifier_internal(proxy, n, false); |
244 | assert(r >= 0); | |
25db9ebe SH |
245 | } |
246 | proxy->ioeventfd_started = false; | |
b36e3914 | 247 | error_report("%s: failed. Fallback to a userspace (slower).", __func__); |
25db9ebe SH |
248 | } |
249 | ||
b36e3914 | 250 | static void virtio_pci_stop_ioeventfd(VirtIOPCIProxy *proxy) |
25db9ebe | 251 | { |
b36e3914 | 252 | int r; |
25db9ebe SH |
253 | int n; |
254 | ||
255 | if (!proxy->ioeventfd_started) { | |
b36e3914 | 256 | return; |
25db9ebe SH |
257 | } |
258 | ||
259 | for (n = 0; n < VIRTIO_PCI_QUEUE_MAX; n++) { | |
260 | if (!virtio_queue_get_num(proxy->vdev, n)) { | |
261 | continue; | |
262 | } | |
263 | ||
264 | virtio_pci_set_host_notifier_fd_handler(proxy, n, false); | |
b36e3914 MT |
265 | r = virtio_pci_set_host_notifier_internal(proxy, n, false); |
266 | assert(r >= 0); | |
25db9ebe SH |
267 | } |
268 | proxy->ioeventfd_started = false; | |
25db9ebe SH |
269 | } |
270 | ||
8798d6c9 | 271 | void virtio_pci_reset(DeviceState *d) |
7055e687 | 272 | { |
e489030d | 273 | VirtIOPCIProxy *proxy = container_of(d, VirtIOPCIProxy, pci_dev.qdev); |
25db9ebe | 274 | virtio_pci_stop_ioeventfd(proxy); |
7055e687 | 275 | virtio_reset(proxy->vdev); |
aba800a3 | 276 | msix_reset(&proxy->pci_dev); |
25db9ebe | 277 | proxy->flags &= ~VIRTIO_PCI_FLAG_BUS_MASTER_BUG; |
7055e687 MT |
278 | } |
279 | ||
53c25cea PB |
280 | static void virtio_ioport_write(void *opaque, uint32_t addr, uint32_t val) |
281 | { | |
282 | VirtIOPCIProxy *proxy = opaque; | |
283 | VirtIODevice *vdev = proxy->vdev; | |
c227f099 | 284 | target_phys_addr_t pa; |
53c25cea | 285 | |
53c25cea PB |
286 | switch (addr) { |
287 | case VIRTIO_PCI_GUEST_FEATURES: | |
288 | /* Guest does not negotiate properly? We have to assume nothing. */ | |
289 | if (val & (1 << VIRTIO_F_BAD_FEATURE)) { | |
ad0c9332 | 290 | val = vdev->bad_features ? vdev->bad_features(vdev) : 0; |
53c25cea | 291 | } |
ad0c9332 | 292 | virtio_set_features(vdev, val); |
53c25cea PB |
293 | break; |
294 | case VIRTIO_PCI_QUEUE_PFN: | |
c227f099 | 295 | pa = (target_phys_addr_t)val << VIRTIO_PCI_QUEUE_ADDR_SHIFT; |
1b8e9b27 | 296 | if (pa == 0) { |
25db9ebe | 297 | virtio_pci_stop_ioeventfd(proxy); |
1b8e9b27 MT |
298 | virtio_reset(proxy->vdev); |
299 | msix_unuse_all_vectors(&proxy->pci_dev); | |
300 | } | |
7055e687 MT |
301 | else |
302 | virtio_queue_set_addr(vdev, vdev->queue_sel, pa); | |
53c25cea PB |
303 | break; |
304 | case VIRTIO_PCI_QUEUE_SEL: | |
305 | if (val < VIRTIO_PCI_QUEUE_MAX) | |
306 | vdev->queue_sel = val; | |
307 | break; | |
308 | case VIRTIO_PCI_QUEUE_NOTIFY: | |
7157e2e2 SH |
309 | if (val < VIRTIO_PCI_QUEUE_MAX) { |
310 | virtio_queue_notify(vdev, val); | |
311 | } | |
53c25cea PB |
312 | break; |
313 | case VIRTIO_PCI_STATUS: | |
25db9ebe SH |
314 | if (!(val & VIRTIO_CONFIG_S_DRIVER_OK)) { |
315 | virtio_pci_stop_ioeventfd(proxy); | |
316 | } | |
317 | ||
3e607cb5 | 318 | virtio_set_status(vdev, val & 0xFF); |
25db9ebe SH |
319 | |
320 | if (val & VIRTIO_CONFIG_S_DRIVER_OK) { | |
321 | virtio_pci_start_ioeventfd(proxy); | |
322 | } | |
323 | ||
1b8e9b27 MT |
324 | if (vdev->status == 0) { |
325 | virtio_reset(proxy->vdev); | |
326 | msix_unuse_all_vectors(&proxy->pci_dev); | |
327 | } | |
c81131db AG |
328 | |
329 | /* Linux before 2.6.34 sets the device as OK without enabling | |
330 | the PCI device bus master bit. In this case we need to disable | |
331 | some safety checks. */ | |
332 | if ((val & VIRTIO_CONFIG_S_DRIVER_OK) && | |
333 | !(proxy->pci_dev.config[PCI_COMMAND] & PCI_COMMAND_MASTER)) { | |
3dbca8e6 | 334 | proxy->flags |= VIRTIO_PCI_FLAG_BUS_MASTER_BUG; |
c81131db | 335 | } |
53c25cea | 336 | break; |
aba800a3 MT |
337 | case VIRTIO_MSI_CONFIG_VECTOR: |
338 | msix_vector_unuse(&proxy->pci_dev, vdev->config_vector); | |
339 | /* Make it possible for guest to discover an error took place. */ | |
340 | if (msix_vector_use(&proxy->pci_dev, val) < 0) | |
341 | val = VIRTIO_NO_VECTOR; | |
342 | vdev->config_vector = val; | |
343 | break; | |
344 | case VIRTIO_MSI_QUEUE_VECTOR: | |
345 | msix_vector_unuse(&proxy->pci_dev, | |
346 | virtio_queue_vector(vdev, vdev->queue_sel)); | |
347 | /* Make it possible for guest to discover an error took place. */ | |
348 | if (msix_vector_use(&proxy->pci_dev, val) < 0) | |
349 | val = VIRTIO_NO_VECTOR; | |
350 | virtio_queue_set_vector(vdev, vdev->queue_sel, val); | |
351 | break; | |
352 | default: | |
4e02d460 SH |
353 | error_report("%s: unexpected address 0x%x value 0x%x", |
354 | __func__, addr, val); | |
aba800a3 | 355 | break; |
53c25cea PB |
356 | } |
357 | } | |
358 | ||
aba800a3 | 359 | static uint32_t virtio_ioport_read(VirtIOPCIProxy *proxy, uint32_t addr) |
53c25cea | 360 | { |
53c25cea PB |
361 | VirtIODevice *vdev = proxy->vdev; |
362 | uint32_t ret = 0xFFFFFFFF; | |
363 | ||
53c25cea PB |
364 | switch (addr) { |
365 | case VIRTIO_PCI_HOST_FEATURES: | |
8172539d | 366 | ret = proxy->host_features; |
53c25cea PB |
367 | break; |
368 | case VIRTIO_PCI_GUEST_FEATURES: | |
704a76fc | 369 | ret = vdev->guest_features; |
53c25cea PB |
370 | break; |
371 | case VIRTIO_PCI_QUEUE_PFN: | |
372 | ret = virtio_queue_get_addr(vdev, vdev->queue_sel) | |
373 | >> VIRTIO_PCI_QUEUE_ADDR_SHIFT; | |
374 | break; | |
375 | case VIRTIO_PCI_QUEUE_NUM: | |
376 | ret = virtio_queue_get_num(vdev, vdev->queue_sel); | |
377 | break; | |
378 | case VIRTIO_PCI_QUEUE_SEL: | |
379 | ret = vdev->queue_sel; | |
380 | break; | |
381 | case VIRTIO_PCI_STATUS: | |
382 | ret = vdev->status; | |
383 | break; | |
384 | case VIRTIO_PCI_ISR: | |
385 | /* reading from the ISR also clears it. */ | |
386 | ret = vdev->isr; | |
387 | vdev->isr = 0; | |
7055e687 | 388 | qemu_set_irq(proxy->pci_dev.irq[0], 0); |
53c25cea | 389 | break; |
aba800a3 MT |
390 | case VIRTIO_MSI_CONFIG_VECTOR: |
391 | ret = vdev->config_vector; | |
392 | break; | |
393 | case VIRTIO_MSI_QUEUE_VECTOR: | |
394 | ret = virtio_queue_vector(vdev, vdev->queue_sel); | |
395 | break; | |
53c25cea PB |
396 | default: |
397 | break; | |
398 | } | |
399 | ||
400 | return ret; | |
401 | } | |
402 | ||
403 | static uint32_t virtio_pci_config_readb(void *opaque, uint32_t addr) | |
404 | { | |
405 | VirtIOPCIProxy *proxy = opaque; | |
aba800a3 | 406 | uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev); |
aba800a3 MT |
407 | if (addr < config) |
408 | return virtio_ioport_read(proxy, addr); | |
409 | addr -= config; | |
53c25cea PB |
410 | return virtio_config_readb(proxy->vdev, addr); |
411 | } | |
412 | ||
413 | static uint32_t virtio_pci_config_readw(void *opaque, uint32_t addr) | |
414 | { | |
415 | VirtIOPCIProxy *proxy = opaque; | |
aba800a3 | 416 | uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev); |
aba800a3 MT |
417 | if (addr < config) |
418 | return virtio_ioport_read(proxy, addr); | |
419 | addr -= config; | |
53c25cea PB |
420 | return virtio_config_readw(proxy->vdev, addr); |
421 | } | |
422 | ||
423 | static uint32_t virtio_pci_config_readl(void *opaque, uint32_t addr) | |
424 | { | |
425 | VirtIOPCIProxy *proxy = opaque; | |
aba800a3 | 426 | uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev); |
aba800a3 MT |
427 | if (addr < config) |
428 | return virtio_ioport_read(proxy, addr); | |
429 | addr -= config; | |
53c25cea PB |
430 | return virtio_config_readl(proxy->vdev, addr); |
431 | } | |
432 | ||
433 | static void virtio_pci_config_writeb(void *opaque, uint32_t addr, uint32_t val) | |
434 | { | |
435 | VirtIOPCIProxy *proxy = opaque; | |
aba800a3 | 436 | uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev); |
aba800a3 MT |
437 | if (addr < config) { |
438 | virtio_ioport_write(proxy, addr, val); | |
439 | return; | |
440 | } | |
441 | addr -= config; | |
53c25cea PB |
442 | virtio_config_writeb(proxy->vdev, addr, val); |
443 | } | |
444 | ||
445 | static void virtio_pci_config_writew(void *opaque, uint32_t addr, uint32_t val) | |
446 | { | |
447 | VirtIOPCIProxy *proxy = opaque; | |
aba800a3 | 448 | uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev); |
aba800a3 MT |
449 | if (addr < config) { |
450 | virtio_ioport_write(proxy, addr, val); | |
451 | return; | |
452 | } | |
453 | addr -= config; | |
53c25cea PB |
454 | virtio_config_writew(proxy->vdev, addr, val); |
455 | } | |
456 | ||
457 | static void virtio_pci_config_writel(void *opaque, uint32_t addr, uint32_t val) | |
458 | { | |
459 | VirtIOPCIProxy *proxy = opaque; | |
aba800a3 | 460 | uint32_t config = VIRTIO_PCI_CONFIG(&proxy->pci_dev); |
aba800a3 MT |
461 | if (addr < config) { |
462 | virtio_ioport_write(proxy, addr, val); | |
463 | return; | |
464 | } | |
465 | addr -= config; | |
53c25cea PB |
466 | virtio_config_writel(proxy->vdev, addr, val); |
467 | } | |
468 | ||
da146d0a AK |
469 | const MemoryRegionPortio virtio_portio[] = { |
470 | { 0, 0x10000, 1, .write = virtio_pci_config_writeb, }, | |
471 | { 0, 0x10000, 2, .write = virtio_pci_config_writew, }, | |
472 | { 0, 0x10000, 4, .write = virtio_pci_config_writel, }, | |
473 | { 0, 0x10000, 1, .read = virtio_pci_config_readb, }, | |
474 | { 0, 0x10000, 2, .read = virtio_pci_config_readw, }, | |
475 | { 0, 0x10000, 4, .read = virtio_pci_config_readl, }, | |
476 | PORTIO_END_OF_LIST() | |
477 | }; | |
53c25cea | 478 | |
da146d0a AK |
479 | static const MemoryRegionOps virtio_pci_config_ops = { |
480 | .old_portio = virtio_portio, | |
481 | .endianness = DEVICE_LITTLE_ENDIAN, | |
482 | }; | |
aba800a3 MT |
483 | |
484 | static void virtio_write_config(PCIDevice *pci_dev, uint32_t address, | |
485 | uint32_t val, int len) | |
486 | { | |
ed757e14 YV |
487 | VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev); |
488 | ||
1129714f MT |
489 | pci_default_write_config(pci_dev, address, val, len); |
490 | ||
491 | if (range_covers_byte(address, len, PCI_COMMAND) && | |
492 | !(pci_dev->config[PCI_COMMAND] & PCI_COMMAND_MASTER) && | |
493 | !(proxy->flags & VIRTIO_PCI_FLAG_BUS_MASTER_BUG)) { | |
494 | virtio_pci_stop_ioeventfd(proxy); | |
495 | virtio_set_status(proxy->vdev, | |
496 | proxy->vdev->status & ~VIRTIO_CONFIG_S_DRIVER_OK); | |
ed757e14 YV |
497 | } |
498 | ||
85352471 | 499 | msix_write_config(pci_dev, address, val, len); |
53c25cea PB |
500 | } |
501 | ||
6d74ca5a MT |
502 | static unsigned virtio_pci_get_features(void *opaque) |
503 | { | |
8172539d MT |
504 | VirtIOPCIProxy *proxy = opaque; |
505 | return proxy->host_features; | |
6d74ca5a MT |
506 | } |
507 | ||
ade80dc8 MT |
508 | static void virtio_pci_guest_notifier_read(void *opaque) |
509 | { | |
510 | VirtQueue *vq = opaque; | |
511 | EventNotifier *n = virtio_queue_get_guest_notifier(vq); | |
512 | if (event_notifier_test_and_clear(n)) { | |
513 | virtio_irq(vq); | |
514 | } | |
515 | } | |
516 | ||
517 | static int virtio_pci_set_guest_notifier(void *opaque, int n, bool assign) | |
518 | { | |
519 | VirtIOPCIProxy *proxy = opaque; | |
520 | VirtQueue *vq = virtio_get_queue(proxy->vdev, n); | |
521 | EventNotifier *notifier = virtio_queue_get_guest_notifier(vq); | |
522 | ||
523 | if (assign) { | |
524 | int r = event_notifier_init(notifier, 0); | |
525 | if (r < 0) { | |
526 | return r; | |
527 | } | |
528 | qemu_set_fd_handler(event_notifier_get_fd(notifier), | |
529 | virtio_pci_guest_notifier_read, NULL, vq); | |
530 | } else { | |
531 | qemu_set_fd_handler(event_notifier_get_fd(notifier), | |
532 | NULL, NULL, NULL); | |
533 | event_notifier_cleanup(notifier); | |
534 | } | |
535 | ||
536 | return 0; | |
537 | } | |
538 | ||
5430a28f MT |
539 | static bool virtio_pci_query_guest_notifiers(void *opaque) |
540 | { | |
541 | VirtIOPCIProxy *proxy = opaque; | |
542 | return msix_enabled(&proxy->pci_dev); | |
543 | } | |
544 | ||
54dd9321 MT |
545 | static int virtio_pci_set_guest_notifiers(void *opaque, bool assign) |
546 | { | |
547 | VirtIOPCIProxy *proxy = opaque; | |
548 | VirtIODevice *vdev = proxy->vdev; | |
549 | int r, n; | |
550 | ||
551 | for (n = 0; n < VIRTIO_PCI_QUEUE_MAX; n++) { | |
552 | if (!virtio_queue_get_num(vdev, n)) { | |
553 | break; | |
554 | } | |
555 | ||
556 | r = virtio_pci_set_guest_notifier(opaque, n, assign); | |
557 | if (r < 0) { | |
558 | goto assign_error; | |
559 | } | |
560 | } | |
561 | ||
562 | return 0; | |
563 | ||
564 | assign_error: | |
565 | /* We get here on assignment failure. Recover by undoing for VQs 0 .. n. */ | |
566 | while (--n >= 0) { | |
567 | virtio_pci_set_guest_notifier(opaque, n, !assign); | |
568 | } | |
569 | return r; | |
570 | } | |
571 | ||
ade80dc8 MT |
572 | static int virtio_pci_set_host_notifier(void *opaque, int n, bool assign) |
573 | { | |
574 | VirtIOPCIProxy *proxy = opaque; | |
25db9ebe SH |
575 | |
576 | /* Stop using ioeventfd for virtqueue kick if the device starts using host | |
577 | * notifiers. This makes it easy to avoid stepping on each others' toes. | |
578 | */ | |
579 | proxy->ioeventfd_disabled = assign; | |
ade80dc8 | 580 | if (assign) { |
25db9ebe SH |
581 | virtio_pci_stop_ioeventfd(proxy); |
582 | } | |
583 | /* We don't need to start here: it's not needed because backend | |
584 | * currently only stops on status change away from ok, | |
585 | * reset, vmstop and such. If we do add code to start here, | |
586 | * need to check vmstate, device state etc. */ | |
587 | return virtio_pci_set_host_notifier_internal(proxy, n, assign); | |
588 | } | |
589 | ||
590 | static void virtio_pci_vmstate_change(void *opaque, bool running) | |
591 | { | |
592 | VirtIOPCIProxy *proxy = opaque; | |
593 | ||
594 | if (running) { | |
89c473fd MT |
595 | /* Try to find out if the guest has bus master disabled, but is |
596 | in ready state. Then we have a buggy guest OS. */ | |
597 | if ((proxy->vdev->status & VIRTIO_CONFIG_S_DRIVER_OK) && | |
598 | !(proxy->pci_dev.config[PCI_COMMAND] & PCI_COMMAND_MASTER)) { | |
599 | proxy->flags |= VIRTIO_PCI_FLAG_BUS_MASTER_BUG; | |
600 | } | |
25db9ebe | 601 | virtio_pci_start_ioeventfd(proxy); |
ade80dc8 | 602 | } else { |
25db9ebe | 603 | virtio_pci_stop_ioeventfd(proxy); |
ade80dc8 | 604 | } |
ade80dc8 MT |
605 | } |
606 | ||
53c25cea | 607 | static const VirtIOBindings virtio_pci_bindings = { |
ff24bd58 MT |
608 | .notify = virtio_pci_notify, |
609 | .save_config = virtio_pci_save_config, | |
610 | .load_config = virtio_pci_load_config, | |
611 | .save_queue = virtio_pci_save_queue, | |
612 | .load_queue = virtio_pci_load_queue, | |
6d74ca5a | 613 | .get_features = virtio_pci_get_features, |
5430a28f | 614 | .query_guest_notifiers = virtio_pci_query_guest_notifiers, |
ade80dc8 | 615 | .set_host_notifier = virtio_pci_set_host_notifier, |
54dd9321 | 616 | .set_guest_notifiers = virtio_pci_set_guest_notifiers, |
25db9ebe | 617 | .vmstate_change = virtio_pci_vmstate_change, |
53c25cea PB |
618 | }; |
619 | ||
befeac45 | 620 | void virtio_init_pci(VirtIOPCIProxy *proxy, VirtIODevice *vdev) |
53c25cea PB |
621 | { |
622 | uint8_t *config; | |
623 | uint32_t size; | |
624 | ||
625 | proxy->vdev = vdev; | |
626 | ||
627 | config = proxy->pci_dev.config; | |
53c25cea | 628 | |
e75ccf2c IY |
629 | if (proxy->class_code) { |
630 | pci_config_set_class(config, proxy->class_code); | |
631 | } | |
ad3d11e6 HKR |
632 | pci_set_word(config + PCI_SUBSYSTEM_VENDOR_ID, |
633 | pci_get_word(config + PCI_VENDOR_ID)); | |
634 | pci_set_word(config + PCI_SUBSYSTEM_ID, vdev->device_id); | |
635 | config[PCI_INTERRUPT_PIN] = 1; | |
53c25cea | 636 | |
95524ae8 AK |
637 | memory_region_init(&proxy->msix_bar, "virtio-msix", 4096); |
638 | if (vdev->nvectors && !msix_init(&proxy->pci_dev, vdev->nvectors, | |
639 | &proxy->msix_bar, 1, 0)) { | |
e824b2cc AK |
640 | pci_register_bar(&proxy->pci_dev, 1, PCI_BASE_ADDRESS_SPACE_MEMORY, |
641 | &proxy->msix_bar); | |
aba800a3 MT |
642 | } else |
643 | vdev->nvectors = 0; | |
644 | ||
ed757e14 YV |
645 | proxy->pci_dev.config_write = virtio_write_config; |
646 | ||
aba800a3 | 647 | size = VIRTIO_PCI_REGION_SIZE(&proxy->pci_dev) + vdev->config_len; |
53c25cea PB |
648 | if (size & (size-1)) |
649 | size = 1 << qemu_fls(size); | |
650 | ||
da146d0a AK |
651 | memory_region_init_io(&proxy->bar, &virtio_pci_config_ops, proxy, |
652 | "virtio-pci", size); | |
e824b2cc AK |
653 | pci_register_bar(&proxy->pci_dev, 0, PCI_BASE_ADDRESS_SPACE_IO, |
654 | &proxy->bar); | |
53c25cea | 655 | |
25db9ebe SH |
656 | if (!kvm_has_many_ioeventfds()) { |
657 | proxy->flags &= ~VIRTIO_PCI_FLAG_USE_IOEVENTFD; | |
658 | } | |
659 | ||
53c25cea | 660 | virtio_bind_device(vdev, &virtio_pci_bindings, proxy); |
8172539d MT |
661 | proxy->host_features |= 0x1 << VIRTIO_F_NOTIFY_ON_EMPTY; |
662 | proxy->host_features |= 0x1 << VIRTIO_F_BAD_FEATURE; | |
663 | proxy->host_features = vdev->get_features(vdev, proxy->host_features); | |
53c25cea PB |
664 | } |
665 | ||
81a322d4 | 666 | static int virtio_blk_init_pci(PCIDevice *pci_dev) |
53c25cea PB |
667 | { |
668 | VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev); | |
669 | VirtIODevice *vdev; | |
670 | ||
ab73ff29 GH |
671 | if (proxy->class_code != PCI_CLASS_STORAGE_SCSI && |
672 | proxy->class_code != PCI_CLASS_STORAGE_OTHER) | |
673 | proxy->class_code = PCI_CLASS_STORAGE_SCSI; | |
53c25cea | 674 | |
a8686a9b MA |
675 | vdev = virtio_blk_init(&pci_dev->qdev, &proxy->block, |
676 | &proxy->block_serial); | |
ac0c14d7 MA |
677 | if (!vdev) { |
678 | return -1; | |
679 | } | |
177539e0 | 680 | vdev->nvectors = proxy->nvectors; |
e75ccf2c | 681 | virtio_init_pci(proxy, vdev); |
177539e0 GH |
682 | /* make the actual value visible */ |
683 | proxy->nvectors = vdev->nvectors; | |
81a322d4 | 684 | return 0; |
21d58b57 MM |
685 | } |
686 | ||
0f457d91 MT |
687 | static int virtio_exit_pci(PCIDevice *pci_dev) |
688 | { | |
da146d0a | 689 | VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev); |
95524ae8 | 690 | int r; |
da146d0a AK |
691 | |
692 | memory_region_destroy(&proxy->bar); | |
95524ae8 AK |
693 | r = msix_uninit(pci_dev, &proxy->msix_bar); |
694 | memory_region_destroy(&proxy->msix_bar); | |
695 | return r; | |
0f457d91 MT |
696 | } |
697 | ||
56a14938 GH |
698 | static int virtio_blk_exit_pci(PCIDevice *pci_dev) |
699 | { | |
700 | VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev); | |
701 | ||
25db9ebe | 702 | virtio_pci_stop_ioeventfd(proxy); |
9d0d3138 | 703 | virtio_blk_exit(proxy->vdev); |
f8b6cc00 | 704 | blockdev_mark_auto_del(proxy->block.bs); |
0f457d91 | 705 | return virtio_exit_pci(pci_dev); |
56a14938 GH |
706 | } |
707 | ||
98b19252 | 708 | static int virtio_serial_init_pci(PCIDevice *pci_dev) |
21d58b57 | 709 | { |
d6beee99 | 710 | VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev); |
85c2c735 MM |
711 | VirtIODevice *vdev; |
712 | ||
d6beee99 GH |
713 | if (proxy->class_code != PCI_CLASS_COMMUNICATION_OTHER && |
714 | proxy->class_code != PCI_CLASS_DISPLAY_OTHER && /* qemu 0.10 */ | |
715 | proxy->class_code != PCI_CLASS_OTHERS) /* qemu-kvm */ | |
716 | proxy->class_code = PCI_CLASS_COMMUNICATION_OTHER; | |
717 | ||
6b331efb | 718 | vdev = virtio_serial_init(&pci_dev->qdev, &proxy->serial); |
25fe3654 AS |
719 | if (!vdev) { |
720 | return -1; | |
721 | } | |
573fb60c | 722 | vdev->nvectors = proxy->nvectors == DEV_NVECTORS_UNSPECIFIED |
6b331efb | 723 | ? proxy->serial.max_virtserial_ports + 1 |
573fb60c | 724 | : proxy->nvectors; |
e75ccf2c | 725 | virtio_init_pci(proxy, vdev); |
a1829205 | 726 | proxy->nvectors = vdev->nvectors; |
81a322d4 | 727 | return 0; |
53c25cea PB |
728 | } |
729 | ||
8b53a865 AS |
730 | static int virtio_serial_exit_pci(PCIDevice *pci_dev) |
731 | { | |
732 | VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev); | |
733 | ||
32059220 | 734 | virtio_pci_stop_ioeventfd(proxy); |
8b53a865 AS |
735 | virtio_serial_exit(proxy->vdev); |
736 | return virtio_exit_pci(pci_dev); | |
737 | } | |
738 | ||
81a322d4 | 739 | static int virtio_net_init_pci(PCIDevice *pci_dev) |
53c25cea PB |
740 | { |
741 | VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev); | |
742 | VirtIODevice *vdev; | |
743 | ||
f0c07c7c | 744 | vdev = virtio_net_init(&pci_dev->qdev, &proxy->nic, &proxy->net); |
a1e0fea5 | 745 | |
97b15621 | 746 | vdev->nvectors = proxy->nvectors; |
e75ccf2c | 747 | virtio_init_pci(proxy, vdev); |
a1e0fea5 GH |
748 | |
749 | /* make the actual value visible */ | |
750 | proxy->nvectors = vdev->nvectors; | |
81a322d4 | 751 | return 0; |
53c25cea PB |
752 | } |
753 | ||
97b15621 GH |
754 | static int virtio_net_exit_pci(PCIDevice *pci_dev) |
755 | { | |
756 | VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev); | |
757 | ||
25db9ebe | 758 | virtio_pci_stop_ioeventfd(proxy); |
97b15621 GH |
759 | virtio_net_exit(proxy->vdev); |
760 | return virtio_exit_pci(pci_dev); | |
761 | } | |
762 | ||
81a322d4 | 763 | static int virtio_balloon_init_pci(PCIDevice *pci_dev) |
53c25cea PB |
764 | { |
765 | VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev); | |
766 | VirtIODevice *vdev; | |
767 | ||
768 | vdev = virtio_balloon_init(&pci_dev->qdev); | |
f76f6655 AS |
769 | if (!vdev) { |
770 | return -1; | |
771 | } | |
e75ccf2c | 772 | virtio_init_pci(proxy, vdev); |
81a322d4 | 773 | return 0; |
53c25cea PB |
774 | } |
775 | ||
855d7e25 AS |
776 | static int virtio_balloon_exit_pci(PCIDevice *pci_dev) |
777 | { | |
778 | VirtIOPCIProxy *proxy = DO_UPCAST(VirtIOPCIProxy, pci_dev, pci_dev); | |
779 | ||
780 | virtio_pci_stop_ioeventfd(proxy); | |
781 | virtio_balloon_exit(proxy->vdev); | |
782 | return virtio_exit_pci(pci_dev); | |
783 | } | |
784 | ||
0aab0d3a GH |
785 | static PCIDeviceInfo virtio_info[] = { |
786 | { | |
787 | .qdev.name = "virtio-blk-pci", | |
779206de | 788 | .qdev.alias = "virtio-blk", |
0aab0d3a GH |
789 | .qdev.size = sizeof(VirtIOPCIProxy), |
790 | .init = virtio_blk_init_pci, | |
56a14938 | 791 | .exit = virtio_blk_exit_pci, |
e75ccf2c IY |
792 | .vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET, |
793 | .device_id = PCI_DEVICE_ID_VIRTIO_BLOCK, | |
794 | .revision = VIRTIO_PCI_ABI_VERSION, | |
795 | .class_id = PCI_CLASS_STORAGE_SCSI, | |
ab73ff29 | 796 | .qdev.props = (Property[]) { |
72c61d0b | 797 | DEFINE_PROP_HEX32("class", VirtIOPCIProxy, class_code, 0), |
428c149b | 798 | DEFINE_BLOCK_PROPERTIES(VirtIOPCIProxy, block), |
a8686a9b | 799 | DEFINE_PROP_STRING("serial", VirtIOPCIProxy, block_serial), |
25db9ebe SH |
800 | DEFINE_PROP_BIT("ioeventfd", VirtIOPCIProxy, flags, |
801 | VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT, true), | |
177539e0 | 802 | DEFINE_PROP_UINT32("vectors", VirtIOPCIProxy, nvectors, 2), |
8172539d | 803 | DEFINE_VIRTIO_BLK_FEATURES(VirtIOPCIProxy, host_features), |
72c61d0b | 804 | DEFINE_PROP_END_OF_LIST(), |
ab73ff29 | 805 | }, |
e489030d | 806 | .qdev.reset = virtio_pci_reset, |
0aab0d3a | 807 | },{ |
a1e0fea5 | 808 | .qdev.name = "virtio-net-pci", |
29f82b37 | 809 | .qdev.alias = "virtio-net", |
a1e0fea5 GH |
810 | .qdev.size = sizeof(VirtIOPCIProxy), |
811 | .init = virtio_net_init_pci, | |
97b15621 | 812 | .exit = virtio_net_exit_pci, |
5ee8ad71 | 813 | .romfile = "pxe-virtio.rom", |
e75ccf2c IY |
814 | .vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET, |
815 | .device_id = PCI_DEVICE_ID_VIRTIO_NET, | |
816 | .revision = VIRTIO_PCI_ABI_VERSION, | |
817 | .class_id = PCI_CLASS_NETWORK_ETHERNET, | |
a1e0fea5 | 818 | .qdev.props = (Property[]) { |
25db9ebe SH |
819 | DEFINE_PROP_BIT("ioeventfd", VirtIOPCIProxy, flags, |
820 | VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT, false), | |
97b15621 | 821 | DEFINE_PROP_UINT32("vectors", VirtIOPCIProxy, nvectors, 3), |
8172539d | 822 | DEFINE_VIRTIO_NET_FEATURES(VirtIOPCIProxy, host_features), |
97b15621 | 823 | DEFINE_NIC_PROPERTIES(VirtIOPCIProxy, nic), |
f0c07c7c AW |
824 | DEFINE_PROP_UINT32("x-txtimer", VirtIOPCIProxy, |
825 | net.txtimer, TX_TIMER_INTERVAL), | |
e3f30488 AW |
826 | DEFINE_PROP_INT32("x-txburst", VirtIOPCIProxy, |
827 | net.txburst, TX_BURST), | |
a697a334 | 828 | DEFINE_PROP_STRING("tx", VirtIOPCIProxy, net.tx), |
72c61d0b | 829 | DEFINE_PROP_END_OF_LIST(), |
a1e0fea5 | 830 | }, |
e489030d | 831 | .qdev.reset = virtio_pci_reset, |
0aab0d3a | 832 | },{ |
98b19252 AS |
833 | .qdev.name = "virtio-serial-pci", |
834 | .qdev.alias = "virtio-serial", | |
0aab0d3a | 835 | .qdev.size = sizeof(VirtIOPCIProxy), |
98b19252 | 836 | .init = virtio_serial_init_pci, |
8b53a865 | 837 | .exit = virtio_serial_exit_pci, |
e75ccf2c IY |
838 | .vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET, |
839 | .device_id = PCI_DEVICE_ID_VIRTIO_CONSOLE, | |
840 | .revision = VIRTIO_PCI_ABI_VERSION, | |
841 | .class_id = PCI_CLASS_COMMUNICATION_OTHER, | |
d6beee99 | 842 | .qdev.props = (Property[]) { |
32059220 AS |
843 | DEFINE_PROP_BIT("ioeventfd", VirtIOPCIProxy, flags, |
844 | VIRTIO_PCI_FLAG_USE_IOEVENTFD_BIT, true), | |
573fb60c AS |
845 | DEFINE_PROP_UINT32("vectors", VirtIOPCIProxy, nvectors, |
846 | DEV_NVECTORS_UNSPECIFIED), | |
72c61d0b | 847 | DEFINE_PROP_HEX32("class", VirtIOPCIProxy, class_code, 0), |
8172539d | 848 | DEFINE_VIRTIO_COMMON_FEATURES(VirtIOPCIProxy, host_features), |
6b331efb AS |
849 | DEFINE_PROP_UINT32("max_ports", VirtIOPCIProxy, |
850 | serial.max_virtserial_ports, 31), | |
72c61d0b | 851 | DEFINE_PROP_END_OF_LIST(), |
d6beee99 | 852 | }, |
e489030d | 853 | .qdev.reset = virtio_pci_reset, |
0aab0d3a GH |
854 | },{ |
855 | .qdev.name = "virtio-balloon-pci", | |
29f82b37 | 856 | .qdev.alias = "virtio-balloon", |
0aab0d3a GH |
857 | .qdev.size = sizeof(VirtIOPCIProxy), |
858 | .init = virtio_balloon_init_pci, | |
855d7e25 | 859 | .exit = virtio_balloon_exit_pci, |
e75ccf2c IY |
860 | .vendor_id = PCI_VENDOR_ID_REDHAT_QUMRANET, |
861 | .device_id = PCI_DEVICE_ID_VIRTIO_BALLOON, | |
862 | .revision = VIRTIO_PCI_ABI_VERSION, | |
863 | .class_id = PCI_CLASS_MEMORY_RAM, | |
8172539d MT |
864 | .qdev.props = (Property[]) { |
865 | DEFINE_VIRTIO_COMMON_FEATURES(VirtIOPCIProxy, host_features), | |
866 | DEFINE_PROP_END_OF_LIST(), | |
867 | }, | |
e489030d | 868 | .qdev.reset = virtio_pci_reset, |
0aab0d3a GH |
869 | },{ |
870 | /* end of list */ | |
871 | } | |
872 | }; | |
873 | ||
53c25cea PB |
874 | static void virtio_pci_register_devices(void) |
875 | { | |
0aab0d3a | 876 | pci_qdev_register_many(virtio_info); |
53c25cea PB |
877 | } |
878 | ||
879 | device_init(virtio_pci_register_devices) |