]>
Commit | Line | Data |
---|---|---|
967f97fa AL |
1 | /* |
2 | * Virtio Support | |
3 | * | |
4 | * Copyright IBM, Corp. 2007 | |
5 | * | |
6 | * Authors: | |
7 | * Anthony Liguori <aliguori@us.ibm.com> | |
8 | * | |
9 | * This work is licensed under the terms of the GNU GPL, version 2. See | |
10 | * the COPYING file in the top-level directory. | |
11 | * | |
12 | */ | |
13 | ||
14 | #include <inttypes.h> | |
967f97fa | 15 | |
64979a4d | 16 | #include "trace.h" |
967f97fa AL |
17 | #include "virtio.h" |
18 | #include "sysemu.h" | |
19 | ||
f46f15bc AL |
20 | /* The alignment to use between consumer and producer parts of vring. |
21 | * x86 pagesize again. */ | |
22 | #define VIRTIO_PCI_VRING_ALIGN 4096 | |
23 | ||
967f97fa AL |
24 | /* QEMU doesn't strictly need write barriers since everything runs in |
25 | * lock-step. We'll leave the calls to wmb() in though to make it obvious for | |
26 | * KVM or if kqemu gets SMP support. | |
79758e95 MT |
27 | * In any case, we must prevent the compiler from reordering the code. |
28 | * TODO: we likely need some rmb()/mb() as well. | |
967f97fa | 29 | */ |
79758e95 MT |
30 | |
31 | #define wmb() __asm__ __volatile__("": : :"memory") | |
967f97fa AL |
32 | |
33 | typedef struct VRingDesc | |
34 | { | |
35 | uint64_t addr; | |
36 | uint32_t len; | |
37 | uint16_t flags; | |
38 | uint16_t next; | |
39 | } VRingDesc; | |
40 | ||
41 | typedef struct VRingAvail | |
42 | { | |
43 | uint16_t flags; | |
44 | uint16_t idx; | |
45 | uint16_t ring[0]; | |
46 | } VRingAvail; | |
47 | ||
48 | typedef struct VRingUsedElem | |
49 | { | |
50 | uint32_t id; | |
51 | uint32_t len; | |
52 | } VRingUsedElem; | |
53 | ||
54 | typedef struct VRingUsed | |
55 | { | |
56 | uint16_t flags; | |
57 | uint16_t idx; | |
58 | VRingUsedElem ring[0]; | |
59 | } VRingUsed; | |
60 | ||
61 | typedef struct VRing | |
62 | { | |
63 | unsigned int num; | |
c227f099 AL |
64 | target_phys_addr_t desc; |
65 | target_phys_addr_t avail; | |
66 | target_phys_addr_t used; | |
967f97fa AL |
67 | } VRing; |
68 | ||
69 | struct VirtQueue | |
70 | { | |
71 | VRing vring; | |
c227f099 | 72 | target_phys_addr_t pa; |
967f97fa AL |
73 | uint16_t last_avail_idx; |
74 | int inuse; | |
7055e687 | 75 | uint16_t vector; |
967f97fa | 76 | void (*handle_output)(VirtIODevice *vdev, VirtQueue *vq); |
1cbdabe2 MT |
77 | VirtIODevice *vdev; |
78 | EventNotifier guest_notifier; | |
79 | EventNotifier host_notifier; | |
967f97fa AL |
80 | }; |
81 | ||
967f97fa | 82 | /* virt queue functions */ |
53c25cea | 83 | static void virtqueue_init(VirtQueue *vq) |
967f97fa | 84 | { |
c227f099 | 85 | target_phys_addr_t pa = vq->pa; |
53c25cea | 86 | |
967f97fa AL |
87 | vq->vring.desc = pa; |
88 | vq->vring.avail = pa + vq->vring.num * sizeof(VRingDesc); | |
f46f15bc AL |
89 | vq->vring.used = vring_align(vq->vring.avail + |
90 | offsetof(VRingAvail, ring[vq->vring.num]), | |
91 | VIRTIO_PCI_VRING_ALIGN); | |
967f97fa AL |
92 | } |
93 | ||
c227f099 | 94 | static inline uint64_t vring_desc_addr(target_phys_addr_t desc_pa, int i) |
967f97fa | 95 | { |
c227f099 | 96 | target_phys_addr_t pa; |
5774cf98 | 97 | pa = desc_pa + sizeof(VRingDesc) * i + offsetof(VRingDesc, addr); |
967f97fa AL |
98 | return ldq_phys(pa); |
99 | } | |
100 | ||
c227f099 | 101 | static inline uint32_t vring_desc_len(target_phys_addr_t desc_pa, int i) |
967f97fa | 102 | { |
c227f099 | 103 | target_phys_addr_t pa; |
5774cf98 | 104 | pa = desc_pa + sizeof(VRingDesc) * i + offsetof(VRingDesc, len); |
967f97fa AL |
105 | return ldl_phys(pa); |
106 | } | |
107 | ||
c227f099 | 108 | static inline uint16_t vring_desc_flags(target_phys_addr_t desc_pa, int i) |
967f97fa | 109 | { |
c227f099 | 110 | target_phys_addr_t pa; |
5774cf98 | 111 | pa = desc_pa + sizeof(VRingDesc) * i + offsetof(VRingDesc, flags); |
967f97fa AL |
112 | return lduw_phys(pa); |
113 | } | |
114 | ||
c227f099 | 115 | static inline uint16_t vring_desc_next(target_phys_addr_t desc_pa, int i) |
967f97fa | 116 | { |
c227f099 | 117 | target_phys_addr_t pa; |
5774cf98 | 118 | pa = desc_pa + sizeof(VRingDesc) * i + offsetof(VRingDesc, next); |
967f97fa AL |
119 | return lduw_phys(pa); |
120 | } | |
121 | ||
122 | static inline uint16_t vring_avail_flags(VirtQueue *vq) | |
123 | { | |
c227f099 | 124 | target_phys_addr_t pa; |
967f97fa AL |
125 | pa = vq->vring.avail + offsetof(VRingAvail, flags); |
126 | return lduw_phys(pa); | |
127 | } | |
128 | ||
129 | static inline uint16_t vring_avail_idx(VirtQueue *vq) | |
130 | { | |
c227f099 | 131 | target_phys_addr_t pa; |
967f97fa AL |
132 | pa = vq->vring.avail + offsetof(VRingAvail, idx); |
133 | return lduw_phys(pa); | |
134 | } | |
135 | ||
136 | static inline uint16_t vring_avail_ring(VirtQueue *vq, int i) | |
137 | { | |
c227f099 | 138 | target_phys_addr_t pa; |
967f97fa AL |
139 | pa = vq->vring.avail + offsetof(VRingAvail, ring[i]); |
140 | return lduw_phys(pa); | |
141 | } | |
142 | ||
143 | static inline void vring_used_ring_id(VirtQueue *vq, int i, uint32_t val) | |
144 | { | |
c227f099 | 145 | target_phys_addr_t pa; |
967f97fa AL |
146 | pa = vq->vring.used + offsetof(VRingUsed, ring[i].id); |
147 | stl_phys(pa, val); | |
148 | } | |
149 | ||
150 | static inline void vring_used_ring_len(VirtQueue *vq, int i, uint32_t val) | |
151 | { | |
c227f099 | 152 | target_phys_addr_t pa; |
967f97fa AL |
153 | pa = vq->vring.used + offsetof(VRingUsed, ring[i].len); |
154 | stl_phys(pa, val); | |
155 | } | |
156 | ||
157 | static uint16_t vring_used_idx(VirtQueue *vq) | |
158 | { | |
c227f099 | 159 | target_phys_addr_t pa; |
967f97fa AL |
160 | pa = vq->vring.used + offsetof(VRingUsed, idx); |
161 | return lduw_phys(pa); | |
162 | } | |
163 | ||
164 | static inline void vring_used_idx_increment(VirtQueue *vq, uint16_t val) | |
165 | { | |
c227f099 | 166 | target_phys_addr_t pa; |
967f97fa AL |
167 | pa = vq->vring.used + offsetof(VRingUsed, idx); |
168 | stw_phys(pa, vring_used_idx(vq) + val); | |
169 | } | |
170 | ||
171 | static inline void vring_used_flags_set_bit(VirtQueue *vq, int mask) | |
172 | { | |
c227f099 | 173 | target_phys_addr_t pa; |
967f97fa AL |
174 | pa = vq->vring.used + offsetof(VRingUsed, flags); |
175 | stw_phys(pa, lduw_phys(pa) | mask); | |
176 | } | |
177 | ||
178 | static inline void vring_used_flags_unset_bit(VirtQueue *vq, int mask) | |
179 | { | |
c227f099 | 180 | target_phys_addr_t pa; |
967f97fa AL |
181 | pa = vq->vring.used + offsetof(VRingUsed, flags); |
182 | stw_phys(pa, lduw_phys(pa) & ~mask); | |
183 | } | |
184 | ||
185 | void virtio_queue_set_notification(VirtQueue *vq, int enable) | |
186 | { | |
187 | if (enable) | |
188 | vring_used_flags_unset_bit(vq, VRING_USED_F_NO_NOTIFY); | |
189 | else | |
190 | vring_used_flags_set_bit(vq, VRING_USED_F_NO_NOTIFY); | |
191 | } | |
192 | ||
193 | int virtio_queue_ready(VirtQueue *vq) | |
194 | { | |
195 | return vq->vring.avail != 0; | |
196 | } | |
197 | ||
198 | int virtio_queue_empty(VirtQueue *vq) | |
199 | { | |
200 | return vring_avail_idx(vq) == vq->last_avail_idx; | |
201 | } | |
202 | ||
203 | void virtqueue_fill(VirtQueue *vq, const VirtQueueElement *elem, | |
204 | unsigned int len, unsigned int idx) | |
205 | { | |
206 | unsigned int offset; | |
207 | int i; | |
208 | ||
64979a4d SH |
209 | trace_virtqueue_fill(vq, elem, len, idx); |
210 | ||
967f97fa AL |
211 | offset = 0; |
212 | for (i = 0; i < elem->in_num; i++) { | |
213 | size_t size = MIN(len - offset, elem->in_sg[i].iov_len); | |
214 | ||
26b258e1 AL |
215 | cpu_physical_memory_unmap(elem->in_sg[i].iov_base, |
216 | elem->in_sg[i].iov_len, | |
217 | 1, size); | |
967f97fa | 218 | |
26b258e1 | 219 | offset += elem->in_sg[i].iov_len; |
967f97fa AL |
220 | } |
221 | ||
26b258e1 AL |
222 | for (i = 0; i < elem->out_num; i++) |
223 | cpu_physical_memory_unmap(elem->out_sg[i].iov_base, | |
224 | elem->out_sg[i].iov_len, | |
225 | 0, elem->out_sg[i].iov_len); | |
226 | ||
967f97fa AL |
227 | idx = (idx + vring_used_idx(vq)) % vq->vring.num; |
228 | ||
229 | /* Get a pointer to the next entry in the used ring. */ | |
230 | vring_used_ring_id(vq, idx, elem->index); | |
231 | vring_used_ring_len(vq, idx, len); | |
232 | } | |
233 | ||
234 | void virtqueue_flush(VirtQueue *vq, unsigned int count) | |
235 | { | |
236 | /* Make sure buffer is written before we update index. */ | |
237 | wmb(); | |
64979a4d | 238 | trace_virtqueue_flush(vq, count); |
967f97fa AL |
239 | vring_used_idx_increment(vq, count); |
240 | vq->inuse -= count; | |
241 | } | |
242 | ||
243 | void virtqueue_push(VirtQueue *vq, const VirtQueueElement *elem, | |
244 | unsigned int len) | |
245 | { | |
246 | virtqueue_fill(vq, elem, len, 0); | |
247 | virtqueue_flush(vq, 1); | |
248 | } | |
249 | ||
250 | static int virtqueue_num_heads(VirtQueue *vq, unsigned int idx) | |
251 | { | |
252 | uint16_t num_heads = vring_avail_idx(vq) - idx; | |
253 | ||
254 | /* Check it isn't doing very strange things with descriptor numbers. */ | |
bb6834cf AL |
255 | if (num_heads > vq->vring.num) { |
256 | fprintf(stderr, "Guest moved used index from %u to %u", | |
257 | idx, vring_avail_idx(vq)); | |
258 | exit(1); | |
259 | } | |
967f97fa AL |
260 | |
261 | return num_heads; | |
262 | } | |
263 | ||
264 | static unsigned int virtqueue_get_head(VirtQueue *vq, unsigned int idx) | |
265 | { | |
266 | unsigned int head; | |
267 | ||
268 | /* Grab the next descriptor number they're advertising, and increment | |
269 | * the index we've seen. */ | |
270 | head = vring_avail_ring(vq, idx % vq->vring.num); | |
271 | ||
272 | /* If their number is silly, that's a fatal mistake. */ | |
bb6834cf AL |
273 | if (head >= vq->vring.num) { |
274 | fprintf(stderr, "Guest says index %u is available", head); | |
275 | exit(1); | |
276 | } | |
967f97fa AL |
277 | |
278 | return head; | |
279 | } | |
280 | ||
c227f099 | 281 | static unsigned virtqueue_next_desc(target_phys_addr_t desc_pa, |
5774cf98 | 282 | unsigned int i, unsigned int max) |
967f97fa AL |
283 | { |
284 | unsigned int next; | |
285 | ||
286 | /* If this descriptor says it doesn't chain, we're done. */ | |
5774cf98 MM |
287 | if (!(vring_desc_flags(desc_pa, i) & VRING_DESC_F_NEXT)) |
288 | return max; | |
967f97fa AL |
289 | |
290 | /* Check they're not leading us off end of descriptors. */ | |
5774cf98 | 291 | next = vring_desc_next(desc_pa, i); |
967f97fa AL |
292 | /* Make sure compiler knows to grab that: we don't want it changing! */ |
293 | wmb(); | |
294 | ||
5774cf98 | 295 | if (next >= max) { |
bb6834cf AL |
296 | fprintf(stderr, "Desc next is %u", next); |
297 | exit(1); | |
298 | } | |
967f97fa AL |
299 | |
300 | return next; | |
301 | } | |
302 | ||
303 | int virtqueue_avail_bytes(VirtQueue *vq, int in_bytes, int out_bytes) | |
304 | { | |
efeea6d0 MM |
305 | unsigned int idx; |
306 | int total_bufs, in_total, out_total; | |
967f97fa AL |
307 | |
308 | idx = vq->last_avail_idx; | |
309 | ||
efeea6d0 | 310 | total_bufs = in_total = out_total = 0; |
967f97fa | 311 | while (virtqueue_num_heads(vq, idx)) { |
efeea6d0 | 312 | unsigned int max, num_bufs, indirect = 0; |
c227f099 | 313 | target_phys_addr_t desc_pa; |
967f97fa AL |
314 | int i; |
315 | ||
efeea6d0 MM |
316 | max = vq->vring.num; |
317 | num_bufs = total_bufs; | |
967f97fa | 318 | i = virtqueue_get_head(vq, idx++); |
efeea6d0 MM |
319 | desc_pa = vq->vring.desc; |
320 | ||
321 | if (vring_desc_flags(desc_pa, i) & VRING_DESC_F_INDIRECT) { | |
322 | if (vring_desc_len(desc_pa, i) % sizeof(VRingDesc)) { | |
323 | fprintf(stderr, "Invalid size for indirect buffer table\n"); | |
324 | exit(1); | |
325 | } | |
326 | ||
327 | /* If we've got too many, that implies a descriptor loop. */ | |
328 | if (num_bufs >= max) { | |
329 | fprintf(stderr, "Looped descriptor"); | |
330 | exit(1); | |
331 | } | |
332 | ||
333 | /* loop over the indirect descriptor table */ | |
334 | indirect = 1; | |
335 | max = vring_desc_len(desc_pa, i) / sizeof(VRingDesc); | |
336 | num_bufs = i = 0; | |
337 | desc_pa = vring_desc_addr(desc_pa, i); | |
338 | } | |
339 | ||
967f97fa AL |
340 | do { |
341 | /* If we've got too many, that implies a descriptor loop. */ | |
5774cf98 | 342 | if (++num_bufs > max) { |
bb6834cf AL |
343 | fprintf(stderr, "Looped descriptor"); |
344 | exit(1); | |
345 | } | |
967f97fa | 346 | |
5774cf98 | 347 | if (vring_desc_flags(desc_pa, i) & VRING_DESC_F_WRITE) { |
967f97fa | 348 | if (in_bytes > 0 && |
5774cf98 | 349 | (in_total += vring_desc_len(desc_pa, i)) >= in_bytes) |
967f97fa AL |
350 | return 1; |
351 | } else { | |
352 | if (out_bytes > 0 && | |
5774cf98 | 353 | (out_total += vring_desc_len(desc_pa, i)) >= out_bytes) |
967f97fa AL |
354 | return 1; |
355 | } | |
5774cf98 | 356 | } while ((i = virtqueue_next_desc(desc_pa, i, max)) != max); |
efeea6d0 MM |
357 | |
358 | if (!indirect) | |
359 | total_bufs = num_bufs; | |
360 | else | |
361 | total_bufs++; | |
967f97fa AL |
362 | } |
363 | ||
364 | return 0; | |
365 | } | |
366 | ||
42fb2e07 KW |
367 | void virtqueue_map_sg(struct iovec *sg, target_phys_addr_t *addr, |
368 | size_t num_sg, int is_write) | |
369 | { | |
370 | unsigned int i; | |
371 | target_phys_addr_t len; | |
372 | ||
373 | for (i = 0; i < num_sg; i++) { | |
374 | len = sg[i].iov_len; | |
375 | sg[i].iov_base = cpu_physical_memory_map(addr[i], &len, is_write); | |
376 | if (sg[i].iov_base == NULL || len != sg[i].iov_len) { | |
377 | fprintf(stderr, "virtio: trying to map MMIO memory\n"); | |
378 | exit(1); | |
379 | } | |
380 | } | |
381 | } | |
382 | ||
967f97fa AL |
383 | int virtqueue_pop(VirtQueue *vq, VirtQueueElement *elem) |
384 | { | |
5774cf98 | 385 | unsigned int i, head, max; |
c227f099 | 386 | target_phys_addr_t desc_pa = vq->vring.desc; |
967f97fa AL |
387 | |
388 | if (!virtqueue_num_heads(vq, vq->last_avail_idx)) | |
389 | return 0; | |
390 | ||
391 | /* When we start there are none of either input nor output. */ | |
392 | elem->out_num = elem->in_num = 0; | |
393 | ||
5774cf98 MM |
394 | max = vq->vring.num; |
395 | ||
967f97fa | 396 | i = head = virtqueue_get_head(vq, vq->last_avail_idx++); |
efeea6d0 MM |
397 | |
398 | if (vring_desc_flags(desc_pa, i) & VRING_DESC_F_INDIRECT) { | |
399 | if (vring_desc_len(desc_pa, i) % sizeof(VRingDesc)) { | |
400 | fprintf(stderr, "Invalid size for indirect buffer table\n"); | |
401 | exit(1); | |
402 | } | |
403 | ||
404 | /* loop over the indirect descriptor table */ | |
405 | max = vring_desc_len(desc_pa, i) / sizeof(VRingDesc); | |
406 | desc_pa = vring_desc_addr(desc_pa, i); | |
407 | i = 0; | |
408 | } | |
409 | ||
42fb2e07 | 410 | /* Collect all the descriptors */ |
967f97fa AL |
411 | do { |
412 | struct iovec *sg; | |
413 | ||
5774cf98 MM |
414 | if (vring_desc_flags(desc_pa, i) & VRING_DESC_F_WRITE) { |
415 | elem->in_addr[elem->in_num] = vring_desc_addr(desc_pa, i); | |
967f97fa | 416 | sg = &elem->in_sg[elem->in_num++]; |
42fb2e07 KW |
417 | } else { |
418 | elem->out_addr[elem->out_num] = vring_desc_addr(desc_pa, i); | |
967f97fa | 419 | sg = &elem->out_sg[elem->out_num++]; |
42fb2e07 | 420 | } |
967f97fa | 421 | |
5774cf98 | 422 | sg->iov_len = vring_desc_len(desc_pa, i); |
967f97fa AL |
423 | |
424 | /* If we've got too many, that implies a descriptor loop. */ | |
5774cf98 | 425 | if ((elem->in_num + elem->out_num) > max) { |
bb6834cf AL |
426 | fprintf(stderr, "Looped descriptor"); |
427 | exit(1); | |
428 | } | |
5774cf98 | 429 | } while ((i = virtqueue_next_desc(desc_pa, i, max)) != max); |
967f97fa | 430 | |
42fb2e07 KW |
431 | /* Now map what we have collected */ |
432 | virtqueue_map_sg(elem->in_sg, elem->in_addr, elem->in_num, 1); | |
433 | virtqueue_map_sg(elem->out_sg, elem->out_addr, elem->out_num, 0); | |
434 | ||
967f97fa AL |
435 | elem->index = head; |
436 | ||
437 | vq->inuse++; | |
438 | ||
64979a4d | 439 | trace_virtqueue_pop(vq, elem, elem->in_num, elem->out_num); |
967f97fa AL |
440 | return elem->in_num + elem->out_num; |
441 | } | |
442 | ||
443 | /* virtio device */ | |
7055e687 MT |
444 | static void virtio_notify_vector(VirtIODevice *vdev, uint16_t vector) |
445 | { | |
446 | if (vdev->binding->notify) { | |
447 | vdev->binding->notify(vdev->binding_opaque, vector); | |
448 | } | |
449 | } | |
967f97fa | 450 | |
53c25cea | 451 | void virtio_update_irq(VirtIODevice *vdev) |
967f97fa | 452 | { |
7055e687 | 453 | virtio_notify_vector(vdev, VIRTIO_NO_VECTOR); |
967f97fa AL |
454 | } |
455 | ||
53c25cea | 456 | void virtio_reset(void *opaque) |
967f97fa AL |
457 | { |
458 | VirtIODevice *vdev = opaque; | |
459 | int i; | |
460 | ||
e0c472d8 MT |
461 | virtio_set_status(vdev, 0); |
462 | ||
967f97fa AL |
463 | if (vdev->reset) |
464 | vdev->reset(vdev); | |
465 | ||
704a76fc | 466 | vdev->guest_features = 0; |
967f97fa AL |
467 | vdev->queue_sel = 0; |
468 | vdev->status = 0; | |
469 | vdev->isr = 0; | |
7055e687 MT |
470 | vdev->config_vector = VIRTIO_NO_VECTOR; |
471 | virtio_notify_vector(vdev, vdev->config_vector); | |
967f97fa AL |
472 | |
473 | for(i = 0; i < VIRTIO_PCI_QUEUE_MAX; i++) { | |
474 | vdev->vq[i].vring.desc = 0; | |
475 | vdev->vq[i].vring.avail = 0; | |
476 | vdev->vq[i].vring.used = 0; | |
477 | vdev->vq[i].last_avail_idx = 0; | |
53c25cea | 478 | vdev->vq[i].pa = 0; |
7055e687 | 479 | vdev->vq[i].vector = VIRTIO_NO_VECTOR; |
967f97fa AL |
480 | } |
481 | } | |
482 | ||
53c25cea | 483 | uint32_t virtio_config_readb(VirtIODevice *vdev, uint32_t addr) |
967f97fa | 484 | { |
967f97fa AL |
485 | uint8_t val; |
486 | ||
487 | vdev->get_config(vdev, vdev->config); | |
488 | ||
967f97fa AL |
489 | if (addr > (vdev->config_len - sizeof(val))) |
490 | return (uint32_t)-1; | |
491 | ||
492 | memcpy(&val, vdev->config + addr, sizeof(val)); | |
493 | return val; | |
494 | } | |
495 | ||
53c25cea | 496 | uint32_t virtio_config_readw(VirtIODevice *vdev, uint32_t addr) |
967f97fa | 497 | { |
967f97fa AL |
498 | uint16_t val; |
499 | ||
500 | vdev->get_config(vdev, vdev->config); | |
501 | ||
967f97fa AL |
502 | if (addr > (vdev->config_len - sizeof(val))) |
503 | return (uint32_t)-1; | |
504 | ||
505 | memcpy(&val, vdev->config + addr, sizeof(val)); | |
506 | return val; | |
507 | } | |
508 | ||
53c25cea | 509 | uint32_t virtio_config_readl(VirtIODevice *vdev, uint32_t addr) |
967f97fa | 510 | { |
967f97fa AL |
511 | uint32_t val; |
512 | ||
513 | vdev->get_config(vdev, vdev->config); | |
514 | ||
967f97fa AL |
515 | if (addr > (vdev->config_len - sizeof(val))) |
516 | return (uint32_t)-1; | |
517 | ||
518 | memcpy(&val, vdev->config + addr, sizeof(val)); | |
519 | return val; | |
520 | } | |
521 | ||
53c25cea | 522 | void virtio_config_writeb(VirtIODevice *vdev, uint32_t addr, uint32_t data) |
967f97fa | 523 | { |
967f97fa AL |
524 | uint8_t val = data; |
525 | ||
967f97fa AL |
526 | if (addr > (vdev->config_len - sizeof(val))) |
527 | return; | |
528 | ||
529 | memcpy(vdev->config + addr, &val, sizeof(val)); | |
530 | ||
531 | if (vdev->set_config) | |
532 | vdev->set_config(vdev, vdev->config); | |
533 | } | |
534 | ||
53c25cea | 535 | void virtio_config_writew(VirtIODevice *vdev, uint32_t addr, uint32_t data) |
967f97fa | 536 | { |
967f97fa AL |
537 | uint16_t val = data; |
538 | ||
967f97fa AL |
539 | if (addr > (vdev->config_len - sizeof(val))) |
540 | return; | |
541 | ||
542 | memcpy(vdev->config + addr, &val, sizeof(val)); | |
543 | ||
544 | if (vdev->set_config) | |
545 | vdev->set_config(vdev, vdev->config); | |
546 | } | |
547 | ||
53c25cea | 548 | void virtio_config_writel(VirtIODevice *vdev, uint32_t addr, uint32_t data) |
967f97fa | 549 | { |
967f97fa AL |
550 | uint32_t val = data; |
551 | ||
967f97fa AL |
552 | if (addr > (vdev->config_len - sizeof(val))) |
553 | return; | |
554 | ||
555 | memcpy(vdev->config + addr, &val, sizeof(val)); | |
556 | ||
557 | if (vdev->set_config) | |
558 | vdev->set_config(vdev, vdev->config); | |
559 | } | |
560 | ||
c227f099 | 561 | void virtio_queue_set_addr(VirtIODevice *vdev, int n, target_phys_addr_t addr) |
967f97fa | 562 | { |
7055e687 MT |
563 | vdev->vq[n].pa = addr; |
564 | virtqueue_init(&vdev->vq[n]); | |
53c25cea PB |
565 | } |
566 | ||
c227f099 | 567 | target_phys_addr_t virtio_queue_get_addr(VirtIODevice *vdev, int n) |
53c25cea PB |
568 | { |
569 | return vdev->vq[n].pa; | |
570 | } | |
571 | ||
572 | int virtio_queue_get_num(VirtIODevice *vdev, int n) | |
573 | { | |
574 | return vdev->vq[n].vring.num; | |
575 | } | |
967f97fa | 576 | |
53c25cea PB |
577 | void virtio_queue_notify(VirtIODevice *vdev, int n) |
578 | { | |
579 | if (n < VIRTIO_PCI_QUEUE_MAX && vdev->vq[n].vring.desc) { | |
64979a4d | 580 | trace_virtio_queue_notify(vdev, n, &vdev->vq[n]); |
53c25cea | 581 | vdev->vq[n].handle_output(vdev, &vdev->vq[n]); |
967f97fa AL |
582 | } |
583 | } | |
584 | ||
7055e687 MT |
585 | uint16_t virtio_queue_vector(VirtIODevice *vdev, int n) |
586 | { | |
587 | return n < VIRTIO_PCI_QUEUE_MAX ? vdev->vq[n].vector : | |
588 | VIRTIO_NO_VECTOR; | |
589 | } | |
590 | ||
591 | void virtio_queue_set_vector(VirtIODevice *vdev, int n, uint16_t vector) | |
592 | { | |
593 | if (n < VIRTIO_PCI_QUEUE_MAX) | |
594 | vdev->vq[n].vector = vector; | |
595 | } | |
596 | ||
967f97fa AL |
597 | VirtQueue *virtio_add_queue(VirtIODevice *vdev, int queue_size, |
598 | void (*handle_output)(VirtIODevice *, VirtQueue *)) | |
599 | { | |
600 | int i; | |
601 | ||
602 | for (i = 0; i < VIRTIO_PCI_QUEUE_MAX; i++) { | |
603 | if (vdev->vq[i].vring.num == 0) | |
604 | break; | |
605 | } | |
606 | ||
607 | if (i == VIRTIO_PCI_QUEUE_MAX || queue_size > VIRTQUEUE_MAX_SIZE) | |
608 | abort(); | |
609 | ||
610 | vdev->vq[i].vring.num = queue_size; | |
611 | vdev->vq[i].handle_output = handle_output; | |
612 | ||
613 | return &vdev->vq[i]; | |
614 | } | |
615 | ||
1cbdabe2 MT |
616 | void virtio_irq(VirtQueue *vq) |
617 | { | |
64979a4d | 618 | trace_virtio_irq(vq); |
1cbdabe2 MT |
619 | vq->vdev->isr |= 0x01; |
620 | virtio_notify_vector(vq->vdev, vq->vector); | |
621 | } | |
622 | ||
967f97fa AL |
623 | void virtio_notify(VirtIODevice *vdev, VirtQueue *vq) |
624 | { | |
97b83deb AL |
625 | /* Always notify when queue is empty (when feature acknowledge) */ |
626 | if ((vring_avail_flags(vq) & VRING_AVAIL_F_NO_INTERRUPT) && | |
704a76fc | 627 | (!(vdev->guest_features & (1 << VIRTIO_F_NOTIFY_ON_EMPTY)) || |
97b83deb | 628 | (vq->inuse || vring_avail_idx(vq) != vq->last_avail_idx))) |
967f97fa AL |
629 | return; |
630 | ||
64979a4d | 631 | trace_virtio_notify(vdev, vq); |
967f97fa | 632 | vdev->isr |= 0x01; |
7055e687 | 633 | virtio_notify_vector(vdev, vq->vector); |
967f97fa AL |
634 | } |
635 | ||
636 | void virtio_notify_config(VirtIODevice *vdev) | |
637 | { | |
7625162c AL |
638 | if (!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK)) |
639 | return; | |
640 | ||
967f97fa | 641 | vdev->isr |= 0x03; |
7055e687 | 642 | virtio_notify_vector(vdev, vdev->config_vector); |
967f97fa AL |
643 | } |
644 | ||
645 | void virtio_save(VirtIODevice *vdev, QEMUFile *f) | |
646 | { | |
647 | int i; | |
648 | ||
ff24bd58 MT |
649 | if (vdev->binding->save_config) |
650 | vdev->binding->save_config(vdev->binding_opaque, f); | |
967f97fa | 651 | |
967f97fa AL |
652 | qemu_put_8s(f, &vdev->status); |
653 | qemu_put_8s(f, &vdev->isr); | |
654 | qemu_put_be16s(f, &vdev->queue_sel); | |
704a76fc | 655 | qemu_put_be32s(f, &vdev->guest_features); |
967f97fa AL |
656 | qemu_put_be32(f, vdev->config_len); |
657 | qemu_put_buffer(f, vdev->config, vdev->config_len); | |
658 | ||
659 | for (i = 0; i < VIRTIO_PCI_QUEUE_MAX; i++) { | |
660 | if (vdev->vq[i].vring.num == 0) | |
661 | break; | |
662 | } | |
663 | ||
664 | qemu_put_be32(f, i); | |
665 | ||
666 | for (i = 0; i < VIRTIO_PCI_QUEUE_MAX; i++) { | |
667 | if (vdev->vq[i].vring.num == 0) | |
668 | break; | |
669 | ||
670 | qemu_put_be32(f, vdev->vq[i].vring.num); | |
53c25cea | 671 | qemu_put_be64(f, vdev->vq[i].pa); |
967f97fa | 672 | qemu_put_be16s(f, &vdev->vq[i].last_avail_idx); |
ff24bd58 MT |
673 | if (vdev->binding->save_queue) |
674 | vdev->binding->save_queue(vdev->binding_opaque, i, f); | |
967f97fa AL |
675 | } |
676 | } | |
677 | ||
ff24bd58 | 678 | int virtio_load(VirtIODevice *vdev, QEMUFile *f) |
967f97fa | 679 | { |
ff24bd58 | 680 | int num, i, ret; |
6d74ca5a | 681 | uint32_t features; |
8172539d | 682 | uint32_t supported_features = |
6d74ca5a | 683 | vdev->binding->get_features(vdev->binding_opaque); |
258dc7c9 | 684 | uint16_t num_heads; |
967f97fa | 685 | |
ff24bd58 MT |
686 | if (vdev->binding->load_config) { |
687 | ret = vdev->binding->load_config(vdev->binding_opaque, f); | |
688 | if (ret) | |
689 | return ret; | |
690 | } | |
967f97fa | 691 | |
967f97fa AL |
692 | qemu_get_8s(f, &vdev->status); |
693 | qemu_get_8s(f, &vdev->isr); | |
694 | qemu_get_be16s(f, &vdev->queue_sel); | |
6d74ca5a MT |
695 | qemu_get_be32s(f, &features); |
696 | if (features & ~supported_features) { | |
697 | fprintf(stderr, "Features 0x%x unsupported. Allowed features: 0x%x\n", | |
698 | features, supported_features); | |
699 | return -1; | |
700 | } | |
fae054b0 MT |
701 | if (vdev->set_features) |
702 | vdev->set_features(vdev, features); | |
704a76fc | 703 | vdev->guest_features = features; |
967f97fa AL |
704 | vdev->config_len = qemu_get_be32(f); |
705 | qemu_get_buffer(f, vdev->config, vdev->config_len); | |
706 | ||
707 | num = qemu_get_be32(f); | |
708 | ||
709 | for (i = 0; i < num; i++) { | |
710 | vdev->vq[i].vring.num = qemu_get_be32(f); | |
53c25cea | 711 | vdev->vq[i].pa = qemu_get_be64(f); |
967f97fa AL |
712 | qemu_get_be16s(f, &vdev->vq[i].last_avail_idx); |
713 | ||
53c25cea PB |
714 | if (vdev->vq[i].pa) { |
715 | virtqueue_init(&vdev->vq[i]); | |
967f97fa | 716 | } |
258dc7c9 MT |
717 | num_heads = vring_avail_idx(&vdev->vq[i]) - vdev->vq[i].last_avail_idx; |
718 | /* Check it isn't doing very strange things with descriptor numbers. */ | |
719 | if (num_heads > vdev->vq[i].vring.num) { | |
720 | fprintf(stderr, "VQ %d size 0x%x Guest index 0x%x " | |
721 | "inconsistent with Host index 0x%x: delta 0x%x\n", | |
722 | i, vdev->vq[i].vring.num, | |
723 | vring_avail_idx(&vdev->vq[i]), | |
724 | vdev->vq[i].last_avail_idx, num_heads); | |
725 | return -1; | |
726 | } | |
ff24bd58 MT |
727 | if (vdev->binding->load_queue) { |
728 | ret = vdev->binding->load_queue(vdev->binding_opaque, i, f); | |
729 | if (ret) | |
730 | return ret; | |
7055e687 | 731 | } |
967f97fa AL |
732 | } |
733 | ||
7055e687 | 734 | virtio_notify_vector(vdev, VIRTIO_NO_VECTOR); |
ff24bd58 | 735 | return 0; |
967f97fa AL |
736 | } |
737 | ||
b946a153 AL |
738 | void virtio_cleanup(VirtIODevice *vdev) |
739 | { | |
740 | if (vdev->config) | |
741 | qemu_free(vdev->config); | |
742 | qemu_free(vdev->vq); | |
743 | } | |
744 | ||
53c25cea PB |
745 | VirtIODevice *virtio_common_init(const char *name, uint16_t device_id, |
746 | size_t config_size, size_t struct_size) | |
967f97fa AL |
747 | { |
748 | VirtIODevice *vdev; | |
b8193adb | 749 | int i; |
967f97fa | 750 | |
53c25cea | 751 | vdev = qemu_mallocz(struct_size); |
967f97fa | 752 | |
53c25cea | 753 | vdev->device_id = device_id; |
967f97fa AL |
754 | vdev->status = 0; |
755 | vdev->isr = 0; | |
756 | vdev->queue_sel = 0; | |
7055e687 | 757 | vdev->config_vector = VIRTIO_NO_VECTOR; |
967f97fa | 758 | vdev->vq = qemu_mallocz(sizeof(VirtQueue) * VIRTIO_PCI_QUEUE_MAX); |
1cbdabe2 | 759 | for(i = 0; i < VIRTIO_PCI_QUEUE_MAX; i++) { |
b8193adb | 760 | vdev->vq[i].vector = VIRTIO_NO_VECTOR; |
1cbdabe2 MT |
761 | vdev->vq[i].vdev = vdev; |
762 | } | |
967f97fa | 763 | |
967f97fa AL |
764 | vdev->name = name; |
765 | vdev->config_len = config_size; | |
766 | if (vdev->config_len) | |
767 | vdev->config = qemu_mallocz(config_size); | |
768 | else | |
769 | vdev->config = NULL; | |
770 | ||
967f97fa AL |
771 | return vdev; |
772 | } | |
53c25cea PB |
773 | |
774 | void virtio_bind_device(VirtIODevice *vdev, const VirtIOBindings *binding, | |
775 | void *opaque) | |
776 | { | |
777 | vdev->binding = binding; | |
778 | vdev->binding_opaque = opaque; | |
779 | } | |
1cbdabe2 MT |
780 | |
781 | target_phys_addr_t virtio_queue_get_desc_addr(VirtIODevice *vdev, int n) | |
782 | { | |
783 | return vdev->vq[n].vring.desc; | |
784 | } | |
785 | ||
786 | target_phys_addr_t virtio_queue_get_avail_addr(VirtIODevice *vdev, int n) | |
787 | { | |
788 | return vdev->vq[n].vring.avail; | |
789 | } | |
790 | ||
791 | target_phys_addr_t virtio_queue_get_used_addr(VirtIODevice *vdev, int n) | |
792 | { | |
793 | return vdev->vq[n].vring.used; | |
794 | } | |
795 | ||
796 | target_phys_addr_t virtio_queue_get_ring_addr(VirtIODevice *vdev, int n) | |
797 | { | |
798 | return vdev->vq[n].vring.desc; | |
799 | } | |
800 | ||
801 | target_phys_addr_t virtio_queue_get_desc_size(VirtIODevice *vdev, int n) | |
802 | { | |
803 | return sizeof(VRingDesc) * vdev->vq[n].vring.num; | |
804 | } | |
805 | ||
806 | target_phys_addr_t virtio_queue_get_avail_size(VirtIODevice *vdev, int n) | |
807 | { | |
808 | return offsetof(VRingAvail, ring) + | |
2b3af999 | 809 | sizeof(uint64_t) * vdev->vq[n].vring.num; |
1cbdabe2 MT |
810 | } |
811 | ||
812 | target_phys_addr_t virtio_queue_get_used_size(VirtIODevice *vdev, int n) | |
813 | { | |
814 | return offsetof(VRingUsed, ring) + | |
815 | sizeof(VRingUsedElem) * vdev->vq[n].vring.num; | |
816 | } | |
817 | ||
818 | target_phys_addr_t virtio_queue_get_ring_size(VirtIODevice *vdev, int n) | |
819 | { | |
820 | return vdev->vq[n].vring.used - vdev->vq[n].vring.desc + | |
821 | virtio_queue_get_used_size(vdev, n); | |
822 | } | |
823 | ||
824 | uint16_t virtio_queue_get_last_avail_idx(VirtIODevice *vdev, int n) | |
825 | { | |
826 | return vdev->vq[n].last_avail_idx; | |
827 | } | |
828 | ||
829 | void virtio_queue_set_last_avail_idx(VirtIODevice *vdev, int n, uint16_t idx) | |
830 | { | |
831 | vdev->vq[n].last_avail_idx = idx; | |
832 | } | |
833 | ||
834 | VirtQueue *virtio_get_queue(VirtIODevice *vdev, int n) | |
835 | { | |
836 | return vdev->vq + n; | |
837 | } | |
838 | ||
839 | EventNotifier *virtio_queue_get_guest_notifier(VirtQueue *vq) | |
840 | { | |
841 | return &vq->guest_notifier; | |
842 | } | |
843 | EventNotifier *virtio_queue_get_host_notifier(VirtQueue *vq) | |
844 | { | |
845 | return &vq->host_notifier; | |
846 | } |