1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /* Virtio ring implementation.
4 * Copyright 2007 Rusty Russell IBM Corporation
6 #include <linux/virtio.h>
7 #include <linux/virtio_ring.h>
8 #include <linux/virtio_config.h>
9 #include <linux/device.h>
10 #include <linux/slab.h>
11 #include <linux/module.h>
12 #include <linux/hrtimer.h>
13 #include <linux/dma-mapping.h>
17 /* For development, we want to crash whenever the ring is screwed. */
18 #define BAD_RING(_vq, fmt, args...) \
20 dev_err(&(_vq)->vq.vdev->dev, \
21 "%s:"fmt, (_vq)->vq.name, ##args); \
24 /* Caller is supposed to guarantee no reentry. */
25 #define START_USE(_vq) \
28 panic("%s:in_use = %i\n", \
29 (_vq)->vq.name, (_vq)->in_use); \
30 (_vq)->in_use = __LINE__; \
32 #define END_USE(_vq) \
33 do { BUG_ON(!(_vq)->in_use); (_vq)->in_use = 0; } while(0)
34 #define LAST_ADD_TIME_UPDATE(_vq) \
36 ktime_t now = ktime_get(); \
38 /* No kick or get, with .1 second between? Warn. */ \
39 if ((_vq)->last_add_time_valid) \
40 WARN_ON(ktime_to_ms(ktime_sub(now, \
41 (_vq)->last_add_time)) > 100); \
42 (_vq)->last_add_time = now; \
43 (_vq)->last_add_time_valid = true; \
45 #define LAST_ADD_TIME_CHECK(_vq) \
47 if ((_vq)->last_add_time_valid) { \
48 WARN_ON(ktime_to_ms(ktime_sub(ktime_get(), \
49 (_vq)->last_add_time)) > 100); \
52 #define LAST_ADD_TIME_INVALID(_vq) \
53 ((_vq)->last_add_time_valid = false)
55 #define BAD_RING(_vq, fmt, args...) \
57 dev_err(&_vq->vq.vdev->dev, \
58 "%s:"fmt, (_vq)->vq.name, ##args); \
59 (_vq)->broken = true; \
63 #define LAST_ADD_TIME_UPDATE(vq)
64 #define LAST_ADD_TIME_CHECK(vq)
65 #define LAST_ADD_TIME_INVALID(vq)
68 struct vring_desc_state_split
{
69 void *data
; /* Data for callback. */
70 struct vring_desc
*indir_desc
; /* Indirect descriptor, if any. */
73 struct vring_desc_state_packed
{
74 void *data
; /* Data for callback. */
75 struct vring_packed_desc
*indir_desc
; /* Indirect descriptor, if any. */
76 u16 num
; /* Descriptor list length. */
77 u16 next
; /* The next desc state in a list. */
78 u16 last
; /* The last desc state in a list. */
81 struct vring_desc_extra_packed
{
82 dma_addr_t addr
; /* Buffer DMA addr. */
83 u32 len
; /* Buffer length. */
84 u16 flags
; /* Descriptor flags. */
87 struct vring_virtqueue
{
90 /* Is this a packed ring? */
93 /* Is DMA API used? */
96 /* Can we use weak barriers? */
99 /* Other side has made a mess, don't try any more. */
102 /* Host supports indirect buffers */
105 /* Host publishes avail event idx */
108 /* Head of free buffer list. */
109 unsigned int free_head
;
110 /* Number we've added since last sync. */
111 unsigned int num_added
;
113 /* Last used index we've seen. */
117 /* Available for split ring */
119 /* Actual memory layout for this queue. */
122 /* Last written value to avail->flags */
123 u16 avail_flags_shadow
;
126 * Last written value to avail->idx in
129 u16 avail_idx_shadow
;
131 /* Per-descriptor state. */
132 struct vring_desc_state_split
*desc_state
;
134 /* DMA address and size information */
135 dma_addr_t queue_dma_addr
;
136 size_t queue_size_in_bytes
;
139 /* Available for packed ring */
141 /* Actual memory layout for this queue. */
144 struct vring_packed_desc
*desc
;
145 struct vring_packed_desc_event
*driver
;
146 struct vring_packed_desc_event
*device
;
149 /* Driver ring wrap counter. */
150 bool avail_wrap_counter
;
152 /* Device ring wrap counter. */
153 bool used_wrap_counter
;
155 /* Avail used flags. */
156 u16 avail_used_flags
;
158 /* Index of the next avail descriptor. */
162 * Last written value to driver->flags in
165 u16 event_flags_shadow
;
167 /* Per-descriptor state. */
168 struct vring_desc_state_packed
*desc_state
;
169 struct vring_desc_extra_packed
*desc_extra
;
171 /* DMA address and size information */
172 dma_addr_t ring_dma_addr
;
173 dma_addr_t driver_event_dma_addr
;
174 dma_addr_t device_event_dma_addr
;
175 size_t ring_size_in_bytes
;
176 size_t event_size_in_bytes
;
180 /* How to notify other side. FIXME: commonalize hcalls! */
181 bool (*notify
)(struct virtqueue
*vq
);
183 /* DMA, allocation, and size information */
187 /* They're supposed to lock for us. */
190 /* Figure out if their kicks are too delayed. */
191 bool last_add_time_valid
;
192 ktime_t last_add_time
;
201 #define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq)
203 static inline bool virtqueue_use_indirect(struct virtqueue
*_vq
,
204 unsigned int total_sg
)
206 struct vring_virtqueue
*vq
= to_vvq(_vq
);
209 * If the host supports indirect descriptor tables, and we have multiple
210 * buffers, then go indirect. FIXME: tune this threshold
212 return (vq
->indirect
&& total_sg
> 1 && vq
->vq
.num_free
);
216 * Modern virtio devices have feature bits to specify whether they need a
217 * quirk and bypass the IOMMU. If not there, just use the DMA API.
219 * If there, the interaction between virtio and DMA API is messy.
221 * On most systems with virtio, physical addresses match bus addresses,
222 * and it doesn't particularly matter whether we use the DMA API.
224 * On some systems, including Xen and any system with a physical device
225 * that speaks virtio behind a physical IOMMU, we must use the DMA API
226 * for virtio DMA to work at all.
228 * On other systems, including SPARC and PPC64, virtio-pci devices are
229 * enumerated as though they are behind an IOMMU, but the virtio host
230 * ignores the IOMMU, so we must either pretend that the IOMMU isn't
231 * there or somehow map everything as the identity.
233 * For the time being, we preserve historic behavior and bypass the DMA
236 * TODO: install a per-device DMA ops structure that does the right thing
237 * taking into account all the above quirks, and use the DMA API
238 * unconditionally on data path.
241 static bool vring_use_dma_api(struct virtio_device
*vdev
)
243 if (!virtio_has_dma_quirk(vdev
))
246 /* Otherwise, we are left to guess. */
248 * In theory, it's possible to have a buggy QEMU-supposed
249 * emulated Q35 IOMMU and Xen enabled at the same time. On
250 * such a configuration, virtio has never worked and will
251 * not work without an even larger kludge. Instead, enable
252 * the DMA API if we're a Xen guest, which at least allows
253 * all of the sensible Xen configurations to work correctly.
261 size_t virtio_max_dma_size(struct virtio_device
*vdev
)
263 size_t max_segment_size
= SIZE_MAX
;
265 if (vring_use_dma_api(vdev
))
266 max_segment_size
= dma_max_mapping_size(&vdev
->dev
);
268 return max_segment_size
;
270 EXPORT_SYMBOL_GPL(virtio_max_dma_size
);
272 static void *vring_alloc_queue(struct virtio_device
*vdev
, size_t size
,
273 dma_addr_t
*dma_handle
, gfp_t flag
)
275 if (vring_use_dma_api(vdev
)) {
276 return dma_alloc_coherent(vdev
->dev
.parent
, size
,
279 void *queue
= alloc_pages_exact(PAGE_ALIGN(size
), flag
);
282 phys_addr_t phys_addr
= virt_to_phys(queue
);
283 *dma_handle
= (dma_addr_t
)phys_addr
;
286 * Sanity check: make sure we dind't truncate
287 * the address. The only arches I can find that
288 * have 64-bit phys_addr_t but 32-bit dma_addr_t
289 * are certain non-highmem MIPS and x86
290 * configurations, but these configurations
291 * should never allocate physical pages above 32
292 * bits, so this is fine. Just in case, throw a
293 * warning and abort if we end up with an
294 * unrepresentable address.
296 if (WARN_ON_ONCE(*dma_handle
!= phys_addr
)) {
297 free_pages_exact(queue
, PAGE_ALIGN(size
));
305 static void vring_free_queue(struct virtio_device
*vdev
, size_t size
,
306 void *queue
, dma_addr_t dma_handle
)
308 if (vring_use_dma_api(vdev
))
309 dma_free_coherent(vdev
->dev
.parent
, size
, queue
, dma_handle
);
311 free_pages_exact(queue
, PAGE_ALIGN(size
));
315 * The DMA ops on various arches are rather gnarly right now, and
316 * making all of the arch DMA ops work on the vring device itself
317 * is a mess. For now, we use the parent device for DMA ops.
319 static inline struct device
*vring_dma_dev(const struct vring_virtqueue
*vq
)
321 return vq
->vq
.vdev
->dev
.parent
;
324 /* Map one sg entry. */
325 static dma_addr_t
vring_map_one_sg(const struct vring_virtqueue
*vq
,
326 struct scatterlist
*sg
,
327 enum dma_data_direction direction
)
329 if (!vq
->use_dma_api
)
330 return (dma_addr_t
)sg_phys(sg
);
333 * We can't use dma_map_sg, because we don't use scatterlists in
334 * the way it expects (we don't guarantee that the scatterlist
335 * will exist for the lifetime of the mapping).
337 return dma_map_page(vring_dma_dev(vq
),
338 sg_page(sg
), sg
->offset
, sg
->length
,
342 static dma_addr_t
vring_map_single(const struct vring_virtqueue
*vq
,
343 void *cpu_addr
, size_t size
,
344 enum dma_data_direction direction
)
346 if (!vq
->use_dma_api
)
347 return (dma_addr_t
)virt_to_phys(cpu_addr
);
349 return dma_map_single(vring_dma_dev(vq
),
350 cpu_addr
, size
, direction
);
353 static int vring_mapping_error(const struct vring_virtqueue
*vq
,
356 if (!vq
->use_dma_api
)
359 return dma_mapping_error(vring_dma_dev(vq
), addr
);
364 * Split ring specific functions - *_split().
367 static void vring_unmap_one_split(const struct vring_virtqueue
*vq
,
368 struct vring_desc
*desc
)
372 if (!vq
->use_dma_api
)
375 flags
= virtio16_to_cpu(vq
->vq
.vdev
, desc
->flags
);
377 if (flags
& VRING_DESC_F_INDIRECT
) {
378 dma_unmap_single(vring_dma_dev(vq
),
379 virtio64_to_cpu(vq
->vq
.vdev
, desc
->addr
),
380 virtio32_to_cpu(vq
->vq
.vdev
, desc
->len
),
381 (flags
& VRING_DESC_F_WRITE
) ?
382 DMA_FROM_DEVICE
: DMA_TO_DEVICE
);
384 dma_unmap_page(vring_dma_dev(vq
),
385 virtio64_to_cpu(vq
->vq
.vdev
, desc
->addr
),
386 virtio32_to_cpu(vq
->vq
.vdev
, desc
->len
),
387 (flags
& VRING_DESC_F_WRITE
) ?
388 DMA_FROM_DEVICE
: DMA_TO_DEVICE
);
392 static struct vring_desc
*alloc_indirect_split(struct virtqueue
*_vq
,
393 unsigned int total_sg
,
396 struct vring_desc
*desc
;
400 * We require lowmem mappings for the descriptors because
401 * otherwise virt_to_phys will give us bogus addresses in the
404 gfp
&= ~__GFP_HIGHMEM
;
406 desc
= kmalloc_array(total_sg
, sizeof(struct vring_desc
), gfp
);
410 for (i
= 0; i
< total_sg
; i
++)
411 desc
[i
].next
= cpu_to_virtio16(_vq
->vdev
, i
+ 1);
415 static inline int virtqueue_add_split(struct virtqueue
*_vq
,
416 struct scatterlist
*sgs
[],
417 unsigned int total_sg
,
418 unsigned int out_sgs
,
424 struct vring_virtqueue
*vq
= to_vvq(_vq
);
425 struct scatterlist
*sg
;
426 struct vring_desc
*desc
;
427 unsigned int i
, n
, avail
, descs_used
, prev
, err_idx
;
433 BUG_ON(data
== NULL
);
434 BUG_ON(ctx
&& vq
->indirect
);
436 if (unlikely(vq
->broken
)) {
441 LAST_ADD_TIME_UPDATE(vq
);
443 BUG_ON(total_sg
== 0);
445 head
= vq
->free_head
;
447 if (virtqueue_use_indirect(_vq
, total_sg
))
448 desc
= alloc_indirect_split(_vq
, total_sg
, gfp
);
451 WARN_ON_ONCE(total_sg
> vq
->split
.vring
.num
&& !vq
->indirect
);
455 /* Use a single buffer which doesn't continue */
457 /* Set up rest to use this indirect table. */
462 desc
= vq
->split
.vring
.desc
;
464 descs_used
= total_sg
;
467 if (vq
->vq
.num_free
< descs_used
) {
468 pr_debug("Can't add buf len %i - avail = %i\n",
469 descs_used
, vq
->vq
.num_free
);
470 /* FIXME: for historical reasons, we force a notify here if
471 * there are outgoing parts to the buffer. Presumably the
472 * host should service the ring ASAP. */
481 for (n
= 0; n
< out_sgs
; n
++) {
482 for (sg
= sgs
[n
]; sg
; sg
= sg_next(sg
)) {
483 dma_addr_t addr
= vring_map_one_sg(vq
, sg
, DMA_TO_DEVICE
);
484 if (vring_mapping_error(vq
, addr
))
487 desc
[i
].flags
= cpu_to_virtio16(_vq
->vdev
, VRING_DESC_F_NEXT
);
488 desc
[i
].addr
= cpu_to_virtio64(_vq
->vdev
, addr
);
489 desc
[i
].len
= cpu_to_virtio32(_vq
->vdev
, sg
->length
);
491 i
= virtio16_to_cpu(_vq
->vdev
, desc
[i
].next
);
494 for (; n
< (out_sgs
+ in_sgs
); n
++) {
495 for (sg
= sgs
[n
]; sg
; sg
= sg_next(sg
)) {
496 dma_addr_t addr
= vring_map_one_sg(vq
, sg
, DMA_FROM_DEVICE
);
497 if (vring_mapping_error(vq
, addr
))
500 desc
[i
].flags
= cpu_to_virtio16(_vq
->vdev
, VRING_DESC_F_NEXT
| VRING_DESC_F_WRITE
);
501 desc
[i
].addr
= cpu_to_virtio64(_vq
->vdev
, addr
);
502 desc
[i
].len
= cpu_to_virtio32(_vq
->vdev
, sg
->length
);
504 i
= virtio16_to_cpu(_vq
->vdev
, desc
[i
].next
);
507 /* Last one doesn't continue. */
508 desc
[prev
].flags
&= cpu_to_virtio16(_vq
->vdev
, ~VRING_DESC_F_NEXT
);
511 /* Now that the indirect table is filled in, map it. */
512 dma_addr_t addr
= vring_map_single(
513 vq
, desc
, total_sg
* sizeof(struct vring_desc
),
515 if (vring_mapping_error(vq
, addr
))
518 vq
->split
.vring
.desc
[head
].flags
= cpu_to_virtio16(_vq
->vdev
,
519 VRING_DESC_F_INDIRECT
);
520 vq
->split
.vring
.desc
[head
].addr
= cpu_to_virtio64(_vq
->vdev
,
523 vq
->split
.vring
.desc
[head
].len
= cpu_to_virtio32(_vq
->vdev
,
524 total_sg
* sizeof(struct vring_desc
));
527 /* We're using some buffers from the free list. */
528 vq
->vq
.num_free
-= descs_used
;
530 /* Update free pointer */
532 vq
->free_head
= virtio16_to_cpu(_vq
->vdev
,
533 vq
->split
.vring
.desc
[head
].next
);
537 /* Store token and indirect buffer state. */
538 vq
->split
.desc_state
[head
].data
= data
;
540 vq
->split
.desc_state
[head
].indir_desc
= desc
;
542 vq
->split
.desc_state
[head
].indir_desc
= ctx
;
544 /* Put entry in available array (but don't update avail->idx until they
546 avail
= vq
->split
.avail_idx_shadow
& (vq
->split
.vring
.num
- 1);
547 vq
->split
.vring
.avail
->ring
[avail
] = cpu_to_virtio16(_vq
->vdev
, head
);
549 /* Descriptors and available array need to be set before we expose the
550 * new available array entries. */
551 virtio_wmb(vq
->weak_barriers
);
552 vq
->split
.avail_idx_shadow
++;
553 vq
->split
.vring
.avail
->idx
= cpu_to_virtio16(_vq
->vdev
,
554 vq
->split
.avail_idx_shadow
);
557 pr_debug("Added buffer head %i to %p\n", head
, vq
);
560 /* This is very unlikely, but theoretically possible. Kick
562 if (unlikely(vq
->num_added
== (1 << 16) - 1))
575 for (n
= 0; n
< total_sg
; n
++) {
578 vring_unmap_one_split(vq
, &desc
[i
]);
579 i
= virtio16_to_cpu(_vq
->vdev
, desc
[i
].next
);
589 static bool virtqueue_kick_prepare_split(struct virtqueue
*_vq
)
591 struct vring_virtqueue
*vq
= to_vvq(_vq
);
596 /* We need to expose available array entries before checking avail
598 virtio_mb(vq
->weak_barriers
);
600 old
= vq
->split
.avail_idx_shadow
- vq
->num_added
;
601 new = vq
->split
.avail_idx_shadow
;
604 LAST_ADD_TIME_CHECK(vq
);
605 LAST_ADD_TIME_INVALID(vq
);
608 needs_kick
= vring_need_event(virtio16_to_cpu(_vq
->vdev
,
609 vring_avail_event(&vq
->split
.vring
)),
612 needs_kick
= !(vq
->split
.vring
.used
->flags
&
613 cpu_to_virtio16(_vq
->vdev
,
614 VRING_USED_F_NO_NOTIFY
));
620 static void detach_buf_split(struct vring_virtqueue
*vq
, unsigned int head
,
624 __virtio16 nextflag
= cpu_to_virtio16(vq
->vq
.vdev
, VRING_DESC_F_NEXT
);
626 /* Clear data ptr. */
627 vq
->split
.desc_state
[head
].data
= NULL
;
629 /* Put back on free list: unmap first-level descriptors and find end */
632 while (vq
->split
.vring
.desc
[i
].flags
& nextflag
) {
633 vring_unmap_one_split(vq
, &vq
->split
.vring
.desc
[i
]);
634 i
= virtio16_to_cpu(vq
->vq
.vdev
, vq
->split
.vring
.desc
[i
].next
);
638 vring_unmap_one_split(vq
, &vq
->split
.vring
.desc
[i
]);
639 vq
->split
.vring
.desc
[i
].next
= cpu_to_virtio16(vq
->vq
.vdev
,
641 vq
->free_head
= head
;
643 /* Plus final descriptor */
647 struct vring_desc
*indir_desc
=
648 vq
->split
.desc_state
[head
].indir_desc
;
651 /* Free the indirect table, if any, now that it's unmapped. */
655 len
= virtio32_to_cpu(vq
->vq
.vdev
,
656 vq
->split
.vring
.desc
[head
].len
);
658 BUG_ON(!(vq
->split
.vring
.desc
[head
].flags
&
659 cpu_to_virtio16(vq
->vq
.vdev
, VRING_DESC_F_INDIRECT
)));
660 BUG_ON(len
== 0 || len
% sizeof(struct vring_desc
));
662 for (j
= 0; j
< len
/ sizeof(struct vring_desc
); j
++)
663 vring_unmap_one_split(vq
, &indir_desc
[j
]);
666 vq
->split
.desc_state
[head
].indir_desc
= NULL
;
668 *ctx
= vq
->split
.desc_state
[head
].indir_desc
;
672 static inline bool more_used_split(const struct vring_virtqueue
*vq
)
674 return vq
->last_used_idx
!= virtio16_to_cpu(vq
->vq
.vdev
,
675 vq
->split
.vring
.used
->idx
);
678 static void *virtqueue_get_buf_ctx_split(struct virtqueue
*_vq
,
682 struct vring_virtqueue
*vq
= to_vvq(_vq
);
689 if (unlikely(vq
->broken
)) {
694 if (!more_used_split(vq
)) {
695 pr_debug("No more buffers in queue\n");
700 /* Only get used array entries after they have been exposed by host. */
701 virtio_rmb(vq
->weak_barriers
);
703 last_used
= (vq
->last_used_idx
& (vq
->split
.vring
.num
- 1));
704 i
= virtio32_to_cpu(_vq
->vdev
,
705 vq
->split
.vring
.used
->ring
[last_used
].id
);
706 *len
= virtio32_to_cpu(_vq
->vdev
,
707 vq
->split
.vring
.used
->ring
[last_used
].len
);
709 if (unlikely(i
>= vq
->split
.vring
.num
)) {
710 BAD_RING(vq
, "id %u out of range\n", i
);
713 if (unlikely(!vq
->split
.desc_state
[i
].data
)) {
714 BAD_RING(vq
, "id %u is not a head!\n", i
);
718 /* detach_buf_split clears data, so grab it now. */
719 ret
= vq
->split
.desc_state
[i
].data
;
720 detach_buf_split(vq
, i
, ctx
);
722 /* If we expect an interrupt for the next entry, tell host
723 * by writing event index and flush out the write before
724 * the read in the next get_buf call. */
725 if (!(vq
->split
.avail_flags_shadow
& VRING_AVAIL_F_NO_INTERRUPT
))
726 virtio_store_mb(vq
->weak_barriers
,
727 &vring_used_event(&vq
->split
.vring
),
728 cpu_to_virtio16(_vq
->vdev
, vq
->last_used_idx
));
730 LAST_ADD_TIME_INVALID(vq
);
736 static void virtqueue_disable_cb_split(struct virtqueue
*_vq
)
738 struct vring_virtqueue
*vq
= to_vvq(_vq
);
740 if (!(vq
->split
.avail_flags_shadow
& VRING_AVAIL_F_NO_INTERRUPT
)) {
741 vq
->split
.avail_flags_shadow
|= VRING_AVAIL_F_NO_INTERRUPT
;
743 vq
->split
.vring
.avail
->flags
=
744 cpu_to_virtio16(_vq
->vdev
,
745 vq
->split
.avail_flags_shadow
);
749 static unsigned virtqueue_enable_cb_prepare_split(struct virtqueue
*_vq
)
751 struct vring_virtqueue
*vq
= to_vvq(_vq
);
756 /* We optimistically turn back on interrupts, then check if there was
758 /* Depending on the VIRTIO_RING_F_EVENT_IDX feature, we need to
759 * either clear the flags bit or point the event index at the next
760 * entry. Always do both to keep code simple. */
761 if (vq
->split
.avail_flags_shadow
& VRING_AVAIL_F_NO_INTERRUPT
) {
762 vq
->split
.avail_flags_shadow
&= ~VRING_AVAIL_F_NO_INTERRUPT
;
764 vq
->split
.vring
.avail
->flags
=
765 cpu_to_virtio16(_vq
->vdev
,
766 vq
->split
.avail_flags_shadow
);
768 vring_used_event(&vq
->split
.vring
) = cpu_to_virtio16(_vq
->vdev
,
769 last_used_idx
= vq
->last_used_idx
);
771 return last_used_idx
;
774 static bool virtqueue_poll_split(struct virtqueue
*_vq
, unsigned last_used_idx
)
776 struct vring_virtqueue
*vq
= to_vvq(_vq
);
778 return (u16
)last_used_idx
!= virtio16_to_cpu(_vq
->vdev
,
779 vq
->split
.vring
.used
->idx
);
782 static bool virtqueue_enable_cb_delayed_split(struct virtqueue
*_vq
)
784 struct vring_virtqueue
*vq
= to_vvq(_vq
);
789 /* We optimistically turn back on interrupts, then check if there was
791 /* Depending on the VIRTIO_RING_F_USED_EVENT_IDX feature, we need to
792 * either clear the flags bit or point the event index at the next
793 * entry. Always update the event index to keep code simple. */
794 if (vq
->split
.avail_flags_shadow
& VRING_AVAIL_F_NO_INTERRUPT
) {
795 vq
->split
.avail_flags_shadow
&= ~VRING_AVAIL_F_NO_INTERRUPT
;
797 vq
->split
.vring
.avail
->flags
=
798 cpu_to_virtio16(_vq
->vdev
,
799 vq
->split
.avail_flags_shadow
);
801 /* TODO: tune this threshold */
802 bufs
= (u16
)(vq
->split
.avail_idx_shadow
- vq
->last_used_idx
) * 3 / 4;
804 virtio_store_mb(vq
->weak_barriers
,
805 &vring_used_event(&vq
->split
.vring
),
806 cpu_to_virtio16(_vq
->vdev
, vq
->last_used_idx
+ bufs
));
808 if (unlikely((u16
)(virtio16_to_cpu(_vq
->vdev
, vq
->split
.vring
.used
->idx
)
809 - vq
->last_used_idx
) > bufs
)) {
818 static void *virtqueue_detach_unused_buf_split(struct virtqueue
*_vq
)
820 struct vring_virtqueue
*vq
= to_vvq(_vq
);
826 for (i
= 0; i
< vq
->split
.vring
.num
; i
++) {
827 if (!vq
->split
.desc_state
[i
].data
)
829 /* detach_buf_split clears data, so grab it now. */
830 buf
= vq
->split
.desc_state
[i
].data
;
831 detach_buf_split(vq
, i
, NULL
);
832 vq
->split
.avail_idx_shadow
--;
833 vq
->split
.vring
.avail
->idx
= cpu_to_virtio16(_vq
->vdev
,
834 vq
->split
.avail_idx_shadow
);
838 /* That should have freed everything. */
839 BUG_ON(vq
->vq
.num_free
!= vq
->split
.vring
.num
);
845 static struct virtqueue
*vring_create_virtqueue_split(
848 unsigned int vring_align
,
849 struct virtio_device
*vdev
,
853 bool (*notify
)(struct virtqueue
*),
854 void (*callback
)(struct virtqueue
*),
857 struct virtqueue
*vq
;
860 size_t queue_size_in_bytes
;
863 /* We assume num is a power of 2. */
864 if (num
& (num
- 1)) {
865 dev_warn(&vdev
->dev
, "Bad virtqueue length %u\n", num
);
869 /* TODO: allocate each queue chunk individually */
870 for (; num
&& vring_size(num
, vring_align
) > PAGE_SIZE
; num
/= 2) {
871 queue
= vring_alloc_queue(vdev
, vring_size(num
, vring_align
),
873 GFP_KERNEL
|__GFP_NOWARN
|__GFP_ZERO
);
884 /* Try to get a single page. You are my only hope! */
885 queue
= vring_alloc_queue(vdev
, vring_size(num
, vring_align
),
886 &dma_addr
, GFP_KERNEL
|__GFP_ZERO
);
891 queue_size_in_bytes
= vring_size(num
, vring_align
);
892 vring_init(&vring
, num
, queue
, vring_align
);
894 vq
= __vring_new_virtqueue(index
, vring
, vdev
, weak_barriers
, context
,
895 notify
, callback
, name
);
897 vring_free_queue(vdev
, queue_size_in_bytes
, queue
,
902 to_vvq(vq
)->split
.queue_dma_addr
= dma_addr
;
903 to_vvq(vq
)->split
.queue_size_in_bytes
= queue_size_in_bytes
;
904 to_vvq(vq
)->we_own_ring
= true;
911 * Packed ring specific functions - *_packed().
914 static void vring_unmap_state_packed(const struct vring_virtqueue
*vq
,
915 struct vring_desc_extra_packed
*state
)
919 if (!vq
->use_dma_api
)
922 flags
= state
->flags
;
924 if (flags
& VRING_DESC_F_INDIRECT
) {
925 dma_unmap_single(vring_dma_dev(vq
),
926 state
->addr
, state
->len
,
927 (flags
& VRING_DESC_F_WRITE
) ?
928 DMA_FROM_DEVICE
: DMA_TO_DEVICE
);
930 dma_unmap_page(vring_dma_dev(vq
),
931 state
->addr
, state
->len
,
932 (flags
& VRING_DESC_F_WRITE
) ?
933 DMA_FROM_DEVICE
: DMA_TO_DEVICE
);
937 static void vring_unmap_desc_packed(const struct vring_virtqueue
*vq
,
938 struct vring_packed_desc
*desc
)
942 if (!vq
->use_dma_api
)
945 flags
= le16_to_cpu(desc
->flags
);
947 if (flags
& VRING_DESC_F_INDIRECT
) {
948 dma_unmap_single(vring_dma_dev(vq
),
949 le64_to_cpu(desc
->addr
),
950 le32_to_cpu(desc
->len
),
951 (flags
& VRING_DESC_F_WRITE
) ?
952 DMA_FROM_DEVICE
: DMA_TO_DEVICE
);
954 dma_unmap_page(vring_dma_dev(vq
),
955 le64_to_cpu(desc
->addr
),
956 le32_to_cpu(desc
->len
),
957 (flags
& VRING_DESC_F_WRITE
) ?
958 DMA_FROM_DEVICE
: DMA_TO_DEVICE
);
962 static struct vring_packed_desc
*alloc_indirect_packed(unsigned int total_sg
,
965 struct vring_packed_desc
*desc
;
968 * We require lowmem mappings for the descriptors because
969 * otherwise virt_to_phys will give us bogus addresses in the
972 gfp
&= ~__GFP_HIGHMEM
;
974 desc
= kmalloc_array(total_sg
, sizeof(struct vring_packed_desc
), gfp
);
979 static int virtqueue_add_indirect_packed(struct vring_virtqueue
*vq
,
980 struct scatterlist
*sgs
[],
981 unsigned int total_sg
,
982 unsigned int out_sgs
,
987 struct vring_packed_desc
*desc
;
988 struct scatterlist
*sg
;
989 unsigned int i
, n
, err_idx
;
993 head
= vq
->packed
.next_avail_idx
;
994 desc
= alloc_indirect_packed(total_sg
, gfp
);
996 if (unlikely(vq
->vq
.num_free
< 1)) {
997 pr_debug("Can't add buf len 1 - avail = 0\n");
1005 BUG_ON(id
== vq
->packed
.vring
.num
);
1007 for (n
= 0; n
< out_sgs
+ in_sgs
; n
++) {
1008 for (sg
= sgs
[n
]; sg
; sg
= sg_next(sg
)) {
1009 addr
= vring_map_one_sg(vq
, sg
, n
< out_sgs
?
1010 DMA_TO_DEVICE
: DMA_FROM_DEVICE
);
1011 if (vring_mapping_error(vq
, addr
))
1014 desc
[i
].flags
= cpu_to_le16(n
< out_sgs
?
1015 0 : VRING_DESC_F_WRITE
);
1016 desc
[i
].addr
= cpu_to_le64(addr
);
1017 desc
[i
].len
= cpu_to_le32(sg
->length
);
1022 /* Now that the indirect table is filled in, map it. */
1023 addr
= vring_map_single(vq
, desc
,
1024 total_sg
* sizeof(struct vring_packed_desc
),
1026 if (vring_mapping_error(vq
, addr
))
1029 vq
->packed
.vring
.desc
[head
].addr
= cpu_to_le64(addr
);
1030 vq
->packed
.vring
.desc
[head
].len
= cpu_to_le32(total_sg
*
1031 sizeof(struct vring_packed_desc
));
1032 vq
->packed
.vring
.desc
[head
].id
= cpu_to_le16(id
);
1034 if (vq
->use_dma_api
) {
1035 vq
->packed
.desc_extra
[id
].addr
= addr
;
1036 vq
->packed
.desc_extra
[id
].len
= total_sg
*
1037 sizeof(struct vring_packed_desc
);
1038 vq
->packed
.desc_extra
[id
].flags
= VRING_DESC_F_INDIRECT
|
1039 vq
->packed
.avail_used_flags
;
1043 * A driver MUST NOT make the first descriptor in the list
1044 * available before all subsequent descriptors comprising
1045 * the list are made available.
1047 virtio_wmb(vq
->weak_barriers
);
1048 vq
->packed
.vring
.desc
[head
].flags
= cpu_to_le16(VRING_DESC_F_INDIRECT
|
1049 vq
->packed
.avail_used_flags
);
1051 /* We're using some buffers from the free list. */
1052 vq
->vq
.num_free
-= 1;
1054 /* Update free pointer */
1056 if (n
>= vq
->packed
.vring
.num
) {
1058 vq
->packed
.avail_wrap_counter
^= 1;
1059 vq
->packed
.avail_used_flags
^=
1060 1 << VRING_PACKED_DESC_F_AVAIL
|
1061 1 << VRING_PACKED_DESC_F_USED
;
1063 vq
->packed
.next_avail_idx
= n
;
1064 vq
->free_head
= vq
->packed
.desc_state
[id
].next
;
1066 /* Store token and indirect buffer state. */
1067 vq
->packed
.desc_state
[id
].num
= 1;
1068 vq
->packed
.desc_state
[id
].data
= data
;
1069 vq
->packed
.desc_state
[id
].indir_desc
= desc
;
1070 vq
->packed
.desc_state
[id
].last
= id
;
1074 pr_debug("Added buffer head %i to %p\n", head
, vq
);
1082 for (i
= 0; i
< err_idx
; i
++)
1083 vring_unmap_desc_packed(vq
, &desc
[i
]);
1091 static inline int virtqueue_add_packed(struct virtqueue
*_vq
,
1092 struct scatterlist
*sgs
[],
1093 unsigned int total_sg
,
1094 unsigned int out_sgs
,
1095 unsigned int in_sgs
,
1100 struct vring_virtqueue
*vq
= to_vvq(_vq
);
1101 struct vring_packed_desc
*desc
;
1102 struct scatterlist
*sg
;
1103 unsigned int i
, n
, c
, descs_used
, err_idx
;
1104 __le16 head_flags
, flags
;
1105 u16 head
, id
, prev
, curr
, avail_used_flags
;
1109 BUG_ON(data
== NULL
);
1110 BUG_ON(ctx
&& vq
->indirect
);
1112 if (unlikely(vq
->broken
)) {
1117 LAST_ADD_TIME_UPDATE(vq
);
1119 BUG_ON(total_sg
== 0);
1121 if (virtqueue_use_indirect(_vq
, total_sg
))
1122 return virtqueue_add_indirect_packed(vq
, sgs
, total_sg
,
1123 out_sgs
, in_sgs
, data
, gfp
);
1125 head
= vq
->packed
.next_avail_idx
;
1126 avail_used_flags
= vq
->packed
.avail_used_flags
;
1128 WARN_ON_ONCE(total_sg
> vq
->packed
.vring
.num
&& !vq
->indirect
);
1130 desc
= vq
->packed
.vring
.desc
;
1132 descs_used
= total_sg
;
1134 if (unlikely(vq
->vq
.num_free
< descs_used
)) {
1135 pr_debug("Can't add buf len %i - avail = %i\n",
1136 descs_used
, vq
->vq
.num_free
);
1142 BUG_ON(id
== vq
->packed
.vring
.num
);
1146 for (n
= 0; n
< out_sgs
+ in_sgs
; n
++) {
1147 for (sg
= sgs
[n
]; sg
; sg
= sg_next(sg
)) {
1148 dma_addr_t addr
= vring_map_one_sg(vq
, sg
, n
< out_sgs
?
1149 DMA_TO_DEVICE
: DMA_FROM_DEVICE
);
1150 if (vring_mapping_error(vq
, addr
))
1153 flags
= cpu_to_le16(vq
->packed
.avail_used_flags
|
1154 (++c
== total_sg
? 0 : VRING_DESC_F_NEXT
) |
1155 (n
< out_sgs
? 0 : VRING_DESC_F_WRITE
));
1159 desc
[i
].flags
= flags
;
1161 desc
[i
].addr
= cpu_to_le64(addr
);
1162 desc
[i
].len
= cpu_to_le32(sg
->length
);
1163 desc
[i
].id
= cpu_to_le16(id
);
1165 if (unlikely(vq
->use_dma_api
)) {
1166 vq
->packed
.desc_extra
[curr
].addr
= addr
;
1167 vq
->packed
.desc_extra
[curr
].len
= sg
->length
;
1168 vq
->packed
.desc_extra
[curr
].flags
=
1172 curr
= vq
->packed
.desc_state
[curr
].next
;
1174 if ((unlikely(++i
>= vq
->packed
.vring
.num
))) {
1176 vq
->packed
.avail_used_flags
^=
1177 1 << VRING_PACKED_DESC_F_AVAIL
|
1178 1 << VRING_PACKED_DESC_F_USED
;
1184 vq
->packed
.avail_wrap_counter
^= 1;
1186 /* We're using some buffers from the free list. */
1187 vq
->vq
.num_free
-= descs_used
;
1189 /* Update free pointer */
1190 vq
->packed
.next_avail_idx
= i
;
1191 vq
->free_head
= curr
;
1194 vq
->packed
.desc_state
[id
].num
= descs_used
;
1195 vq
->packed
.desc_state
[id
].data
= data
;
1196 vq
->packed
.desc_state
[id
].indir_desc
= ctx
;
1197 vq
->packed
.desc_state
[id
].last
= prev
;
1200 * A driver MUST NOT make the first descriptor in the list
1201 * available before all subsequent descriptors comprising
1202 * the list are made available.
1204 virtio_wmb(vq
->weak_barriers
);
1205 vq
->packed
.vring
.desc
[head
].flags
= head_flags
;
1206 vq
->num_added
+= descs_used
;
1208 pr_debug("Added buffer head %i to %p\n", head
, vq
);
1217 vq
->packed
.avail_used_flags
= avail_used_flags
;
1219 for (n
= 0; n
< total_sg
; n
++) {
1222 vring_unmap_desc_packed(vq
, &desc
[i
]);
1224 if (i
>= vq
->packed
.vring
.num
)
1232 static bool virtqueue_kick_prepare_packed(struct virtqueue
*_vq
)
1234 struct vring_virtqueue
*vq
= to_vvq(_vq
);
1235 u16
new, old
, off_wrap
, flags
, wrap_counter
, event_idx
;
1248 * We need to expose the new flags value before checking notification
1251 virtio_mb(vq
->weak_barriers
);
1253 old
= vq
->packed
.next_avail_idx
- vq
->num_added
;
1254 new = vq
->packed
.next_avail_idx
;
1257 snapshot
.u32
= *(u32
*)vq
->packed
.vring
.device
;
1258 flags
= le16_to_cpu(snapshot
.flags
);
1260 LAST_ADD_TIME_CHECK(vq
);
1261 LAST_ADD_TIME_INVALID(vq
);
1263 if (flags
!= VRING_PACKED_EVENT_FLAG_DESC
) {
1264 needs_kick
= (flags
!= VRING_PACKED_EVENT_FLAG_DISABLE
);
1268 off_wrap
= le16_to_cpu(snapshot
.off_wrap
);
1270 wrap_counter
= off_wrap
>> VRING_PACKED_EVENT_F_WRAP_CTR
;
1271 event_idx
= off_wrap
& ~(1 << VRING_PACKED_EVENT_F_WRAP_CTR
);
1272 if (wrap_counter
!= vq
->packed
.avail_wrap_counter
)
1273 event_idx
-= vq
->packed
.vring
.num
;
1275 needs_kick
= vring_need_event(event_idx
, new, old
);
1281 static void detach_buf_packed(struct vring_virtqueue
*vq
,
1282 unsigned int id
, void **ctx
)
1284 struct vring_desc_state_packed
*state
= NULL
;
1285 struct vring_packed_desc
*desc
;
1286 unsigned int i
, curr
;
1288 state
= &vq
->packed
.desc_state
[id
];
1290 /* Clear data ptr. */
1293 vq
->packed
.desc_state
[state
->last
].next
= vq
->free_head
;
1295 vq
->vq
.num_free
+= state
->num
;
1297 if (unlikely(vq
->use_dma_api
)) {
1299 for (i
= 0; i
< state
->num
; i
++) {
1300 vring_unmap_state_packed(vq
,
1301 &vq
->packed
.desc_extra
[curr
]);
1302 curr
= vq
->packed
.desc_state
[curr
].next
;
1309 /* Free the indirect table, if any, now that it's unmapped. */
1310 desc
= state
->indir_desc
;
1314 if (vq
->use_dma_api
) {
1315 len
= vq
->packed
.desc_extra
[id
].len
;
1316 for (i
= 0; i
< len
/ sizeof(struct vring_packed_desc
);
1318 vring_unmap_desc_packed(vq
, &desc
[i
]);
1321 state
->indir_desc
= NULL
;
1323 *ctx
= state
->indir_desc
;
1327 static inline bool is_used_desc_packed(const struct vring_virtqueue
*vq
,
1328 u16 idx
, bool used_wrap_counter
)
1333 flags
= le16_to_cpu(vq
->packed
.vring
.desc
[idx
].flags
);
1334 avail
= !!(flags
& (1 << VRING_PACKED_DESC_F_AVAIL
));
1335 used
= !!(flags
& (1 << VRING_PACKED_DESC_F_USED
));
1337 return avail
== used
&& used
== used_wrap_counter
;
1340 static inline bool more_used_packed(const struct vring_virtqueue
*vq
)
1342 return is_used_desc_packed(vq
, vq
->last_used_idx
,
1343 vq
->packed
.used_wrap_counter
);
1346 static void *virtqueue_get_buf_ctx_packed(struct virtqueue
*_vq
,
1350 struct vring_virtqueue
*vq
= to_vvq(_vq
);
1356 if (unlikely(vq
->broken
)) {
1361 if (!more_used_packed(vq
)) {
1362 pr_debug("No more buffers in queue\n");
1367 /* Only get used elements after they have been exposed by host. */
1368 virtio_rmb(vq
->weak_barriers
);
1370 last_used
= vq
->last_used_idx
;
1371 id
= le16_to_cpu(vq
->packed
.vring
.desc
[last_used
].id
);
1372 *len
= le32_to_cpu(vq
->packed
.vring
.desc
[last_used
].len
);
1374 if (unlikely(id
>= vq
->packed
.vring
.num
)) {
1375 BAD_RING(vq
, "id %u out of range\n", id
);
1378 if (unlikely(!vq
->packed
.desc_state
[id
].data
)) {
1379 BAD_RING(vq
, "id %u is not a head!\n", id
);
1383 /* detach_buf_packed clears data, so grab it now. */
1384 ret
= vq
->packed
.desc_state
[id
].data
;
1385 detach_buf_packed(vq
, id
, ctx
);
1387 vq
->last_used_idx
+= vq
->packed
.desc_state
[id
].num
;
1388 if (unlikely(vq
->last_used_idx
>= vq
->packed
.vring
.num
)) {
1389 vq
->last_used_idx
-= vq
->packed
.vring
.num
;
1390 vq
->packed
.used_wrap_counter
^= 1;
1394 * If we expect an interrupt for the next entry, tell host
1395 * by writing event index and flush out the write before
1396 * the read in the next get_buf call.
1398 if (vq
->packed
.event_flags_shadow
== VRING_PACKED_EVENT_FLAG_DESC
)
1399 virtio_store_mb(vq
->weak_barriers
,
1400 &vq
->packed
.vring
.driver
->off_wrap
,
1401 cpu_to_le16(vq
->last_used_idx
|
1402 (vq
->packed
.used_wrap_counter
<<
1403 VRING_PACKED_EVENT_F_WRAP_CTR
)));
1405 LAST_ADD_TIME_INVALID(vq
);
1411 static void virtqueue_disable_cb_packed(struct virtqueue
*_vq
)
1413 struct vring_virtqueue
*vq
= to_vvq(_vq
);
1415 if (vq
->packed
.event_flags_shadow
!= VRING_PACKED_EVENT_FLAG_DISABLE
) {
1416 vq
->packed
.event_flags_shadow
= VRING_PACKED_EVENT_FLAG_DISABLE
;
1417 vq
->packed
.vring
.driver
->flags
=
1418 cpu_to_le16(vq
->packed
.event_flags_shadow
);
1422 static unsigned virtqueue_enable_cb_prepare_packed(struct virtqueue
*_vq
)
1424 struct vring_virtqueue
*vq
= to_vvq(_vq
);
1429 * We optimistically turn back on interrupts, then check if there was
1434 vq
->packed
.vring
.driver
->off_wrap
=
1435 cpu_to_le16(vq
->last_used_idx
|
1436 (vq
->packed
.used_wrap_counter
<<
1437 VRING_PACKED_EVENT_F_WRAP_CTR
));
1439 * We need to update event offset and event wrap
1440 * counter first before updating event flags.
1442 virtio_wmb(vq
->weak_barriers
);
1445 if (vq
->packed
.event_flags_shadow
== VRING_PACKED_EVENT_FLAG_DISABLE
) {
1446 vq
->packed
.event_flags_shadow
= vq
->event
?
1447 VRING_PACKED_EVENT_FLAG_DESC
:
1448 VRING_PACKED_EVENT_FLAG_ENABLE
;
1449 vq
->packed
.vring
.driver
->flags
=
1450 cpu_to_le16(vq
->packed
.event_flags_shadow
);
1454 return vq
->last_used_idx
| ((u16
)vq
->packed
.used_wrap_counter
<<
1455 VRING_PACKED_EVENT_F_WRAP_CTR
);
1458 static bool virtqueue_poll_packed(struct virtqueue
*_vq
, u16 off_wrap
)
1460 struct vring_virtqueue
*vq
= to_vvq(_vq
);
1464 wrap_counter
= off_wrap
>> VRING_PACKED_EVENT_F_WRAP_CTR
;
1465 used_idx
= off_wrap
& ~(1 << VRING_PACKED_EVENT_F_WRAP_CTR
);
1467 return is_used_desc_packed(vq
, used_idx
, wrap_counter
);
1470 static bool virtqueue_enable_cb_delayed_packed(struct virtqueue
*_vq
)
1472 struct vring_virtqueue
*vq
= to_vvq(_vq
);
1473 u16 used_idx
, wrap_counter
;
1479 * We optimistically turn back on interrupts, then check if there was
1484 /* TODO: tune this threshold */
1485 bufs
= (vq
->packed
.vring
.num
- vq
->vq
.num_free
) * 3 / 4;
1486 wrap_counter
= vq
->packed
.used_wrap_counter
;
1488 used_idx
= vq
->last_used_idx
+ bufs
;
1489 if (used_idx
>= vq
->packed
.vring
.num
) {
1490 used_idx
-= vq
->packed
.vring
.num
;
1494 vq
->packed
.vring
.driver
->off_wrap
= cpu_to_le16(used_idx
|
1495 (wrap_counter
<< VRING_PACKED_EVENT_F_WRAP_CTR
));
1498 * We need to update event offset and event wrap
1499 * counter first before updating event flags.
1501 virtio_wmb(vq
->weak_barriers
);
1504 if (vq
->packed
.event_flags_shadow
== VRING_PACKED_EVENT_FLAG_DISABLE
) {
1505 vq
->packed
.event_flags_shadow
= vq
->event
?
1506 VRING_PACKED_EVENT_FLAG_DESC
:
1507 VRING_PACKED_EVENT_FLAG_ENABLE
;
1508 vq
->packed
.vring
.driver
->flags
=
1509 cpu_to_le16(vq
->packed
.event_flags_shadow
);
1513 * We need to update event suppression structure first
1514 * before re-checking for more used buffers.
1516 virtio_mb(vq
->weak_barriers
);
1518 if (is_used_desc_packed(vq
,
1520 vq
->packed
.used_wrap_counter
)) {
1529 static void *virtqueue_detach_unused_buf_packed(struct virtqueue
*_vq
)
1531 struct vring_virtqueue
*vq
= to_vvq(_vq
);
1537 for (i
= 0; i
< vq
->packed
.vring
.num
; i
++) {
1538 if (!vq
->packed
.desc_state
[i
].data
)
1540 /* detach_buf clears data, so grab it now. */
1541 buf
= vq
->packed
.desc_state
[i
].data
;
1542 detach_buf_packed(vq
, i
, NULL
);
1546 /* That should have freed everything. */
1547 BUG_ON(vq
->vq
.num_free
!= vq
->packed
.vring
.num
);
1553 static struct virtqueue
*vring_create_virtqueue_packed(
1556 unsigned int vring_align
,
1557 struct virtio_device
*vdev
,
1559 bool may_reduce_num
,
1561 bool (*notify
)(struct virtqueue
*),
1562 void (*callback
)(struct virtqueue
*),
1565 struct vring_virtqueue
*vq
;
1566 struct vring_packed_desc
*ring
;
1567 struct vring_packed_desc_event
*driver
, *device
;
1568 dma_addr_t ring_dma_addr
, driver_event_dma_addr
, device_event_dma_addr
;
1569 size_t ring_size_in_bytes
, event_size_in_bytes
;
1572 ring_size_in_bytes
= num
* sizeof(struct vring_packed_desc
);
1574 ring
= vring_alloc_queue(vdev
, ring_size_in_bytes
,
1576 GFP_KERNEL
|__GFP_NOWARN
|__GFP_ZERO
);
1580 event_size_in_bytes
= sizeof(struct vring_packed_desc_event
);
1582 driver
= vring_alloc_queue(vdev
, event_size_in_bytes
,
1583 &driver_event_dma_addr
,
1584 GFP_KERNEL
|__GFP_NOWARN
|__GFP_ZERO
);
1588 device
= vring_alloc_queue(vdev
, event_size_in_bytes
,
1589 &device_event_dma_addr
,
1590 GFP_KERNEL
|__GFP_NOWARN
|__GFP_ZERO
);
1594 vq
= kmalloc(sizeof(*vq
), GFP_KERNEL
);
1598 vq
->vq
.callback
= callback
;
1601 vq
->vq
.num_free
= num
;
1602 vq
->vq
.index
= index
;
1603 vq
->we_own_ring
= true;
1604 vq
->notify
= notify
;
1605 vq
->weak_barriers
= weak_barriers
;
1607 vq
->last_used_idx
= 0;
1609 vq
->packed_ring
= true;
1610 vq
->use_dma_api
= vring_use_dma_api(vdev
);
1613 vq
->last_add_time_valid
= false;
1616 vq
->indirect
= virtio_has_feature(vdev
, VIRTIO_RING_F_INDIRECT_DESC
) &&
1618 vq
->event
= virtio_has_feature(vdev
, VIRTIO_RING_F_EVENT_IDX
);
1620 if (virtio_has_feature(vdev
, VIRTIO_F_ORDER_PLATFORM
))
1621 vq
->weak_barriers
= false;
1623 vq
->packed
.ring_dma_addr
= ring_dma_addr
;
1624 vq
->packed
.driver_event_dma_addr
= driver_event_dma_addr
;
1625 vq
->packed
.device_event_dma_addr
= device_event_dma_addr
;
1627 vq
->packed
.ring_size_in_bytes
= ring_size_in_bytes
;
1628 vq
->packed
.event_size_in_bytes
= event_size_in_bytes
;
1630 vq
->packed
.vring
.num
= num
;
1631 vq
->packed
.vring
.desc
= ring
;
1632 vq
->packed
.vring
.driver
= driver
;
1633 vq
->packed
.vring
.device
= device
;
1635 vq
->packed
.next_avail_idx
= 0;
1636 vq
->packed
.avail_wrap_counter
= 1;
1637 vq
->packed
.used_wrap_counter
= 1;
1638 vq
->packed
.event_flags_shadow
= 0;
1639 vq
->packed
.avail_used_flags
= 1 << VRING_PACKED_DESC_F_AVAIL
;
1641 vq
->packed
.desc_state
= kmalloc_array(num
,
1642 sizeof(struct vring_desc_state_packed
),
1644 if (!vq
->packed
.desc_state
)
1645 goto err_desc_state
;
1647 memset(vq
->packed
.desc_state
, 0,
1648 num
* sizeof(struct vring_desc_state_packed
));
1650 /* Put everything in free lists. */
1652 for (i
= 0; i
< num
-1; i
++)
1653 vq
->packed
.desc_state
[i
].next
= i
+ 1;
1655 vq
->packed
.desc_extra
= kmalloc_array(num
,
1656 sizeof(struct vring_desc_extra_packed
),
1658 if (!vq
->packed
.desc_extra
)
1659 goto err_desc_extra
;
1661 memset(vq
->packed
.desc_extra
, 0,
1662 num
* sizeof(struct vring_desc_extra_packed
));
1664 /* No callback? Tell other side not to bother us. */
1666 vq
->packed
.event_flags_shadow
= VRING_PACKED_EVENT_FLAG_DISABLE
;
1667 vq
->packed
.vring
.driver
->flags
=
1668 cpu_to_le16(vq
->packed
.event_flags_shadow
);
1671 list_add_tail(&vq
->vq
.list
, &vdev
->vqs
);
1675 kfree(vq
->packed
.desc_state
);
1679 vring_free_queue(vdev
, event_size_in_bytes
, device
, device_event_dma_addr
);
1681 vring_free_queue(vdev
, event_size_in_bytes
, driver
, driver_event_dma_addr
);
1683 vring_free_queue(vdev
, ring_size_in_bytes
, ring
, ring_dma_addr
);
1690 * Generic functions and exported symbols.
1693 static inline int virtqueue_add(struct virtqueue
*_vq
,
1694 struct scatterlist
*sgs
[],
1695 unsigned int total_sg
,
1696 unsigned int out_sgs
,
1697 unsigned int in_sgs
,
1702 struct vring_virtqueue
*vq
= to_vvq(_vq
);
1704 return vq
->packed_ring
? virtqueue_add_packed(_vq
, sgs
, total_sg
,
1705 out_sgs
, in_sgs
, data
, ctx
, gfp
) :
1706 virtqueue_add_split(_vq
, sgs
, total_sg
,
1707 out_sgs
, in_sgs
, data
, ctx
, gfp
);
1711 * virtqueue_add_sgs - expose buffers to other end
1712 * @_vq: the struct virtqueue we're talking about.
1713 * @sgs: array of terminated scatterlists.
1714 * @out_sgs: the number of scatterlists readable by other side
1715 * @in_sgs: the number of scatterlists which are writable (after readable ones)
1716 * @data: the token identifying the buffer.
1717 * @gfp: how to do memory allocations (if necessary).
1719 * Caller must ensure we don't call this with other virtqueue operations
1720 * at the same time (except where noted).
1722 * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
1724 int virtqueue_add_sgs(struct virtqueue
*_vq
,
1725 struct scatterlist
*sgs
[],
1726 unsigned int out_sgs
,
1727 unsigned int in_sgs
,
1731 unsigned int i
, total_sg
= 0;
1733 /* Count them first. */
1734 for (i
= 0; i
< out_sgs
+ in_sgs
; i
++) {
1735 struct scatterlist
*sg
;
1737 for (sg
= sgs
[i
]; sg
; sg
= sg_next(sg
))
1740 return virtqueue_add(_vq
, sgs
, total_sg
, out_sgs
, in_sgs
,
1743 EXPORT_SYMBOL_GPL(virtqueue_add_sgs
);
1746 * virtqueue_add_outbuf - expose output buffers to other end
1747 * @vq: the struct virtqueue we're talking about.
1748 * @sg: scatterlist (must be well-formed and terminated!)
1749 * @num: the number of entries in @sg readable by other side
1750 * @data: the token identifying the buffer.
1751 * @gfp: how to do memory allocations (if necessary).
1753 * Caller must ensure we don't call this with other virtqueue operations
1754 * at the same time (except where noted).
1756 * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
1758 int virtqueue_add_outbuf(struct virtqueue
*vq
,
1759 struct scatterlist
*sg
, unsigned int num
,
1763 return virtqueue_add(vq
, &sg
, num
, 1, 0, data
, NULL
, gfp
);
1765 EXPORT_SYMBOL_GPL(virtqueue_add_outbuf
);
1768 * virtqueue_add_inbuf - expose input buffers to other end
1769 * @vq: the struct virtqueue we're talking about.
1770 * @sg: scatterlist (must be well-formed and terminated!)
1771 * @num: the number of entries in @sg writable by other side
1772 * @data: the token identifying the buffer.
1773 * @gfp: how to do memory allocations (if necessary).
1775 * Caller must ensure we don't call this with other virtqueue operations
1776 * at the same time (except where noted).
1778 * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
1780 int virtqueue_add_inbuf(struct virtqueue
*vq
,
1781 struct scatterlist
*sg
, unsigned int num
,
1785 return virtqueue_add(vq
, &sg
, num
, 0, 1, data
, NULL
, gfp
);
1787 EXPORT_SYMBOL_GPL(virtqueue_add_inbuf
);
1790 * virtqueue_add_inbuf_ctx - expose input buffers to other end
1791 * @vq: the struct virtqueue we're talking about.
1792 * @sg: scatterlist (must be well-formed and terminated!)
1793 * @num: the number of entries in @sg writable by other side
1794 * @data: the token identifying the buffer.
1795 * @ctx: extra context for the token
1796 * @gfp: how to do memory allocations (if necessary).
1798 * Caller must ensure we don't call this with other virtqueue operations
1799 * at the same time (except where noted).
1801 * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
1803 int virtqueue_add_inbuf_ctx(struct virtqueue
*vq
,
1804 struct scatterlist
*sg
, unsigned int num
,
1809 return virtqueue_add(vq
, &sg
, num
, 0, 1, data
, ctx
, gfp
);
1811 EXPORT_SYMBOL_GPL(virtqueue_add_inbuf_ctx
);
1814 * virtqueue_kick_prepare - first half of split virtqueue_kick call.
1815 * @_vq: the struct virtqueue
1817 * Instead of virtqueue_kick(), you can do:
1818 * if (virtqueue_kick_prepare(vq))
1819 * virtqueue_notify(vq);
1821 * This is sometimes useful because the virtqueue_kick_prepare() needs
1822 * to be serialized, but the actual virtqueue_notify() call does not.
1824 bool virtqueue_kick_prepare(struct virtqueue
*_vq
)
1826 struct vring_virtqueue
*vq
= to_vvq(_vq
);
1828 return vq
->packed_ring
? virtqueue_kick_prepare_packed(_vq
) :
1829 virtqueue_kick_prepare_split(_vq
);
1831 EXPORT_SYMBOL_GPL(virtqueue_kick_prepare
);
1834 * virtqueue_notify - second half of split virtqueue_kick call.
1835 * @_vq: the struct virtqueue
1837 * This does not need to be serialized.
1839 * Returns false if host notify failed or queue is broken, otherwise true.
1841 bool virtqueue_notify(struct virtqueue
*_vq
)
1843 struct vring_virtqueue
*vq
= to_vvq(_vq
);
1845 if (unlikely(vq
->broken
))
1848 /* Prod other side to tell it about changes. */
1849 if (!vq
->notify(_vq
)) {
1855 EXPORT_SYMBOL_GPL(virtqueue_notify
);
1858 * virtqueue_kick - update after add_buf
1859 * @vq: the struct virtqueue
1861 * After one or more virtqueue_add_* calls, invoke this to kick
1864 * Caller must ensure we don't call this with other virtqueue
1865 * operations at the same time (except where noted).
1867 * Returns false if kick failed, otherwise true.
1869 bool virtqueue_kick(struct virtqueue
*vq
)
1871 if (virtqueue_kick_prepare(vq
))
1872 return virtqueue_notify(vq
);
1875 EXPORT_SYMBOL_GPL(virtqueue_kick
);
1878 * virtqueue_get_buf - get the next used buffer
1879 * @_vq: the struct virtqueue we're talking about.
1880 * @len: the length written into the buffer
1881 * @ctx: extra context for the token
1883 * If the device wrote data into the buffer, @len will be set to the
1884 * amount written. This means you don't need to clear the buffer
1885 * beforehand to ensure there's no data leakage in the case of short
1888 * Caller must ensure we don't call this with other virtqueue
1889 * operations at the same time (except where noted).
1891 * Returns NULL if there are no used buffers, or the "data" token
1892 * handed to virtqueue_add_*().
1894 void *virtqueue_get_buf_ctx(struct virtqueue
*_vq
, unsigned int *len
,
1897 struct vring_virtqueue
*vq
= to_vvq(_vq
);
1899 return vq
->packed_ring
? virtqueue_get_buf_ctx_packed(_vq
, len
, ctx
) :
1900 virtqueue_get_buf_ctx_split(_vq
, len
, ctx
);
1902 EXPORT_SYMBOL_GPL(virtqueue_get_buf_ctx
);
1904 void *virtqueue_get_buf(struct virtqueue
*_vq
, unsigned int *len
)
1906 return virtqueue_get_buf_ctx(_vq
, len
, NULL
);
1908 EXPORT_SYMBOL_GPL(virtqueue_get_buf
);
1910 * virtqueue_disable_cb - disable callbacks
1911 * @_vq: the struct virtqueue we're talking about.
1913 * Note that this is not necessarily synchronous, hence unreliable and only
1914 * useful as an optimization.
1916 * Unlike other operations, this need not be serialized.
1918 void virtqueue_disable_cb(struct virtqueue
*_vq
)
1920 struct vring_virtqueue
*vq
= to_vvq(_vq
);
1922 if (vq
->packed_ring
)
1923 virtqueue_disable_cb_packed(_vq
);
1925 virtqueue_disable_cb_split(_vq
);
1927 EXPORT_SYMBOL_GPL(virtqueue_disable_cb
);
1930 * virtqueue_enable_cb_prepare - restart callbacks after disable_cb
1931 * @_vq: the struct virtqueue we're talking about.
1933 * This re-enables callbacks; it returns current queue state
1934 * in an opaque unsigned value. This value should be later tested by
1935 * virtqueue_poll, to detect a possible race between the driver checking for
1936 * more work, and enabling callbacks.
1938 * Caller must ensure we don't call this with other virtqueue
1939 * operations at the same time (except where noted).
1941 unsigned virtqueue_enable_cb_prepare(struct virtqueue
*_vq
)
1943 struct vring_virtqueue
*vq
= to_vvq(_vq
);
1945 return vq
->packed_ring
? virtqueue_enable_cb_prepare_packed(_vq
) :
1946 virtqueue_enable_cb_prepare_split(_vq
);
1948 EXPORT_SYMBOL_GPL(virtqueue_enable_cb_prepare
);
1951 * virtqueue_poll - query pending used buffers
1952 * @_vq: the struct virtqueue we're talking about.
1953 * @last_used_idx: virtqueue state (from call to virtqueue_enable_cb_prepare).
1955 * Returns "true" if there are pending used buffers in the queue.
1957 * This does not need to be serialized.
1959 bool virtqueue_poll(struct virtqueue
*_vq
, unsigned last_used_idx
)
1961 struct vring_virtqueue
*vq
= to_vvq(_vq
);
1963 if (unlikely(vq
->broken
))
1966 virtio_mb(vq
->weak_barriers
);
1967 return vq
->packed_ring
? virtqueue_poll_packed(_vq
, last_used_idx
) :
1968 virtqueue_poll_split(_vq
, last_used_idx
);
1970 EXPORT_SYMBOL_GPL(virtqueue_poll
);
1973 * virtqueue_enable_cb - restart callbacks after disable_cb.
1974 * @_vq: the struct virtqueue we're talking about.
1976 * This re-enables callbacks; it returns "false" if there are pending
1977 * buffers in the queue, to detect a possible race between the driver
1978 * checking for more work, and enabling callbacks.
1980 * Caller must ensure we don't call this with other virtqueue
1981 * operations at the same time (except where noted).
1983 bool virtqueue_enable_cb(struct virtqueue
*_vq
)
1985 unsigned last_used_idx
= virtqueue_enable_cb_prepare(_vq
);
1987 return !virtqueue_poll(_vq
, last_used_idx
);
1989 EXPORT_SYMBOL_GPL(virtqueue_enable_cb
);
1992 * virtqueue_enable_cb_delayed - restart callbacks after disable_cb.
1993 * @_vq: the struct virtqueue we're talking about.
1995 * This re-enables callbacks but hints to the other side to delay
1996 * interrupts until most of the available buffers have been processed;
1997 * it returns "false" if there are many pending buffers in the queue,
1998 * to detect a possible race between the driver checking for more work,
1999 * and enabling callbacks.
2001 * Caller must ensure we don't call this with other virtqueue
2002 * operations at the same time (except where noted).
2004 bool virtqueue_enable_cb_delayed(struct virtqueue
*_vq
)
2006 struct vring_virtqueue
*vq
= to_vvq(_vq
);
2008 return vq
->packed_ring
? virtqueue_enable_cb_delayed_packed(_vq
) :
2009 virtqueue_enable_cb_delayed_split(_vq
);
2011 EXPORT_SYMBOL_GPL(virtqueue_enable_cb_delayed
);
2014 * virtqueue_detach_unused_buf - detach first unused buffer
2015 * @_vq: the struct virtqueue we're talking about.
2017 * Returns NULL or the "data" token handed to virtqueue_add_*().
2018 * This is not valid on an active queue; it is useful only for device
2021 void *virtqueue_detach_unused_buf(struct virtqueue
*_vq
)
2023 struct vring_virtqueue
*vq
= to_vvq(_vq
);
2025 return vq
->packed_ring
? virtqueue_detach_unused_buf_packed(_vq
) :
2026 virtqueue_detach_unused_buf_split(_vq
);
2028 EXPORT_SYMBOL_GPL(virtqueue_detach_unused_buf
);
2030 static inline bool more_used(const struct vring_virtqueue
*vq
)
2032 return vq
->packed_ring
? more_used_packed(vq
) : more_used_split(vq
);
2035 irqreturn_t
vring_interrupt(int irq
, void *_vq
)
2037 struct vring_virtqueue
*vq
= to_vvq(_vq
);
2039 if (!more_used(vq
)) {
2040 pr_debug("virtqueue interrupt with no work for %p\n", vq
);
2044 if (unlikely(vq
->broken
))
2047 pr_debug("virtqueue callback for %p (%p)\n", vq
, vq
->vq
.callback
);
2048 if (vq
->vq
.callback
)
2049 vq
->vq
.callback(&vq
->vq
);
2053 EXPORT_SYMBOL_GPL(vring_interrupt
);
2055 /* Only available for split ring */
2056 struct virtqueue
*__vring_new_virtqueue(unsigned int index
,
2058 struct virtio_device
*vdev
,
2061 bool (*notify
)(struct virtqueue
*),
2062 void (*callback
)(struct virtqueue
*),
2066 struct vring_virtqueue
*vq
;
2068 if (virtio_has_feature(vdev
, VIRTIO_F_RING_PACKED
))
2071 vq
= kmalloc(sizeof(*vq
), GFP_KERNEL
);
2075 vq
->packed_ring
= false;
2076 vq
->vq
.callback
= callback
;
2079 vq
->vq
.num_free
= vring
.num
;
2080 vq
->vq
.index
= index
;
2081 vq
->we_own_ring
= false;
2082 vq
->notify
= notify
;
2083 vq
->weak_barriers
= weak_barriers
;
2085 vq
->last_used_idx
= 0;
2087 vq
->use_dma_api
= vring_use_dma_api(vdev
);
2090 vq
->last_add_time_valid
= false;
2093 vq
->indirect
= virtio_has_feature(vdev
, VIRTIO_RING_F_INDIRECT_DESC
) &&
2095 vq
->event
= virtio_has_feature(vdev
, VIRTIO_RING_F_EVENT_IDX
);
2097 if (virtio_has_feature(vdev
, VIRTIO_F_ORDER_PLATFORM
))
2098 vq
->weak_barriers
= false;
2100 vq
->split
.queue_dma_addr
= 0;
2101 vq
->split
.queue_size_in_bytes
= 0;
2103 vq
->split
.vring
= vring
;
2104 vq
->split
.avail_flags_shadow
= 0;
2105 vq
->split
.avail_idx_shadow
= 0;
2107 /* No callback? Tell other side not to bother us. */
2109 vq
->split
.avail_flags_shadow
|= VRING_AVAIL_F_NO_INTERRUPT
;
2111 vq
->split
.vring
.avail
->flags
= cpu_to_virtio16(vdev
,
2112 vq
->split
.avail_flags_shadow
);
2115 vq
->split
.desc_state
= kmalloc_array(vring
.num
,
2116 sizeof(struct vring_desc_state_split
), GFP_KERNEL
);
2117 if (!vq
->split
.desc_state
) {
2122 /* Put everything in free lists. */
2124 for (i
= 0; i
< vring
.num
-1; i
++)
2125 vq
->split
.vring
.desc
[i
].next
= cpu_to_virtio16(vdev
, i
+ 1);
2126 memset(vq
->split
.desc_state
, 0, vring
.num
*
2127 sizeof(struct vring_desc_state_split
));
2129 list_add_tail(&vq
->vq
.list
, &vdev
->vqs
);
2132 EXPORT_SYMBOL_GPL(__vring_new_virtqueue
);
2134 struct virtqueue
*vring_create_virtqueue(
2137 unsigned int vring_align
,
2138 struct virtio_device
*vdev
,
2140 bool may_reduce_num
,
2142 bool (*notify
)(struct virtqueue
*),
2143 void (*callback
)(struct virtqueue
*),
2147 if (virtio_has_feature(vdev
, VIRTIO_F_RING_PACKED
))
2148 return vring_create_virtqueue_packed(index
, num
, vring_align
,
2149 vdev
, weak_barriers
, may_reduce_num
,
2150 context
, notify
, callback
, name
);
2152 return vring_create_virtqueue_split(index
, num
, vring_align
,
2153 vdev
, weak_barriers
, may_reduce_num
,
2154 context
, notify
, callback
, name
);
2156 EXPORT_SYMBOL_GPL(vring_create_virtqueue
);
2158 /* Only available for split ring */
2159 struct virtqueue
*vring_new_virtqueue(unsigned int index
,
2161 unsigned int vring_align
,
2162 struct virtio_device
*vdev
,
2166 bool (*notify
)(struct virtqueue
*vq
),
2167 void (*callback
)(struct virtqueue
*vq
),
2172 if (virtio_has_feature(vdev
, VIRTIO_F_RING_PACKED
))
2175 vring_init(&vring
, num
, pages
, vring_align
);
2176 return __vring_new_virtqueue(index
, vring
, vdev
, weak_barriers
, context
,
2177 notify
, callback
, name
);
2179 EXPORT_SYMBOL_GPL(vring_new_virtqueue
);
2181 void vring_del_virtqueue(struct virtqueue
*_vq
)
2183 struct vring_virtqueue
*vq
= to_vvq(_vq
);
2185 if (vq
->we_own_ring
) {
2186 if (vq
->packed_ring
) {
2187 vring_free_queue(vq
->vq
.vdev
,
2188 vq
->packed
.ring_size_in_bytes
,
2189 vq
->packed
.vring
.desc
,
2190 vq
->packed
.ring_dma_addr
);
2192 vring_free_queue(vq
->vq
.vdev
,
2193 vq
->packed
.event_size_in_bytes
,
2194 vq
->packed
.vring
.driver
,
2195 vq
->packed
.driver_event_dma_addr
);
2197 vring_free_queue(vq
->vq
.vdev
,
2198 vq
->packed
.event_size_in_bytes
,
2199 vq
->packed
.vring
.device
,
2200 vq
->packed
.device_event_dma_addr
);
2202 kfree(vq
->packed
.desc_state
);
2203 kfree(vq
->packed
.desc_extra
);
2205 vring_free_queue(vq
->vq
.vdev
,
2206 vq
->split
.queue_size_in_bytes
,
2207 vq
->split
.vring
.desc
,
2208 vq
->split
.queue_dma_addr
);
2211 if (!vq
->packed_ring
)
2212 kfree(vq
->split
.desc_state
);
2213 list_del(&_vq
->list
);
2216 EXPORT_SYMBOL_GPL(vring_del_virtqueue
);
2218 /* Manipulates transport-specific feature bits. */
2219 void vring_transport_features(struct virtio_device
*vdev
)
2223 for (i
= VIRTIO_TRANSPORT_F_START
; i
< VIRTIO_TRANSPORT_F_END
; i
++) {
2225 case VIRTIO_RING_F_INDIRECT_DESC
:
2227 case VIRTIO_RING_F_EVENT_IDX
:
2229 case VIRTIO_F_VERSION_1
:
2231 case VIRTIO_F_ACCESS_PLATFORM
:
2233 case VIRTIO_F_RING_PACKED
:
2235 case VIRTIO_F_ORDER_PLATFORM
:
2238 /* We don't understand this bit. */
2239 __virtio_clear_bit(vdev
, i
);
2243 EXPORT_SYMBOL_GPL(vring_transport_features
);
2246 * virtqueue_get_vring_size - return the size of the virtqueue's vring
2247 * @_vq: the struct virtqueue containing the vring of interest.
2249 * Returns the size of the vring. This is mainly used for boasting to
2250 * userspace. Unlike other operations, this need not be serialized.
2252 unsigned int virtqueue_get_vring_size(struct virtqueue
*_vq
)
2255 struct vring_virtqueue
*vq
= to_vvq(_vq
);
2257 return vq
->packed_ring
? vq
->packed
.vring
.num
: vq
->split
.vring
.num
;
2259 EXPORT_SYMBOL_GPL(virtqueue_get_vring_size
);
2261 bool virtqueue_is_broken(struct virtqueue
*_vq
)
2263 struct vring_virtqueue
*vq
= to_vvq(_vq
);
2267 EXPORT_SYMBOL_GPL(virtqueue_is_broken
);
2270 * This should prevent the device from being used, allowing drivers to
2271 * recover. You may need to grab appropriate locks to flush.
2273 void virtio_break_device(struct virtio_device
*dev
)
2275 struct virtqueue
*_vq
;
2277 list_for_each_entry(_vq
, &dev
->vqs
, list
) {
2278 struct vring_virtqueue
*vq
= to_vvq(_vq
);
2282 EXPORT_SYMBOL_GPL(virtio_break_device
);
2284 dma_addr_t
virtqueue_get_desc_addr(struct virtqueue
*_vq
)
2286 struct vring_virtqueue
*vq
= to_vvq(_vq
);
2288 BUG_ON(!vq
->we_own_ring
);
2290 if (vq
->packed_ring
)
2291 return vq
->packed
.ring_dma_addr
;
2293 return vq
->split
.queue_dma_addr
;
2295 EXPORT_SYMBOL_GPL(virtqueue_get_desc_addr
);
2297 dma_addr_t
virtqueue_get_avail_addr(struct virtqueue
*_vq
)
2299 struct vring_virtqueue
*vq
= to_vvq(_vq
);
2301 BUG_ON(!vq
->we_own_ring
);
2303 if (vq
->packed_ring
)
2304 return vq
->packed
.driver_event_dma_addr
;
2306 return vq
->split
.queue_dma_addr
+
2307 ((char *)vq
->split
.vring
.avail
- (char *)vq
->split
.vring
.desc
);
2309 EXPORT_SYMBOL_GPL(virtqueue_get_avail_addr
);
2311 dma_addr_t
virtqueue_get_used_addr(struct virtqueue
*_vq
)
2313 struct vring_virtqueue
*vq
= to_vvq(_vq
);
2315 BUG_ON(!vq
->we_own_ring
);
2317 if (vq
->packed_ring
)
2318 return vq
->packed
.device_event_dma_addr
;
2320 return vq
->split
.queue_dma_addr
+
2321 ((char *)vq
->split
.vring
.used
- (char *)vq
->split
.vring
.desc
);
2323 EXPORT_SYMBOL_GPL(virtqueue_get_used_addr
);
2325 /* Only available for split ring */
2326 const struct vring
*virtqueue_get_vring(struct virtqueue
*vq
)
2328 return &to_vvq(vq
)->split
.vring
;
2330 EXPORT_SYMBOL_GPL(virtqueue_get_vring
);
2332 MODULE_LICENSE("GPL");