]>
Commit | Line | Data |
---|---|---|
244ab90e AL |
1 | /* |
2 | * DMA helper functions | |
3 | * | |
bb755f52 | 4 | * Copyright (c) 2009,2020 Red Hat |
244ab90e AL |
5 | * |
6 | * This work is licensed under the terms of the GNU General Public License | |
7 | * (GNU GPL), version 2 or later. | |
8 | */ | |
9 | ||
d38ea87a | 10 | #include "qemu/osdep.h" |
4be74634 | 11 | #include "sysemu/block-backend.h" |
9c17d615 | 12 | #include "sysemu/dma.h" |
243af022 | 13 | #include "trace/trace-root.h" |
1de7afc9 | 14 | #include "qemu/thread.h" |
6a1751b7 | 15 | #include "qemu/main-loop.h" |
740b1759 | 16 | #include "sysemu/cpu-timers.h" |
5fb0a6b5 | 17 | #include "qemu/range.h" |
244ab90e | 18 | |
e5332e63 DG |
19 | /* #define DEBUG_IOMMU */ |
20 | ||
bb755f52 | 21 | MemTxResult dma_memory_set(AddressSpace *as, dma_addr_t addr, |
7a36e42d | 22 | uint8_t c, dma_addr_t len, MemTxAttrs attrs) |
d86a77f8 | 23 | { |
df32fd1c | 24 | dma_barrier(as, DMA_DIRECTION_FROM_DEVICE); |
24addbc7 | 25 | |
75f01c68 | 26 | return address_space_set(as, addr, c, len, attrs); |
d86a77f8 DG |
27 | } |
28 | ||
f487b677 PB |
29 | void qemu_sglist_init(QEMUSGList *qsg, DeviceState *dev, int alloc_hint, |
30 | AddressSpace *as) | |
244ab90e | 31 | { |
b21e2380 | 32 | qsg->sg = g_new(ScatterGatherEntry, alloc_hint); |
244ab90e AL |
33 | qsg->nsg = 0; |
34 | qsg->nalloc = alloc_hint; | |
35 | qsg->size = 0; | |
df32fd1c | 36 | qsg->as = as; |
f487b677 PB |
37 | qsg->dev = dev; |
38 | object_ref(OBJECT(dev)); | |
244ab90e AL |
39 | } |
40 | ||
d3231181 | 41 | void qemu_sglist_add(QEMUSGList *qsg, dma_addr_t base, dma_addr_t len) |
244ab90e AL |
42 | { |
43 | if (qsg->nsg == qsg->nalloc) { | |
44 | qsg->nalloc = 2 * qsg->nalloc + 1; | |
b21e2380 | 45 | qsg->sg = g_renew(ScatterGatherEntry, qsg->sg, qsg->nalloc); |
244ab90e AL |
46 | } |
47 | qsg->sg[qsg->nsg].base = base; | |
48 | qsg->sg[qsg->nsg].len = len; | |
49 | qsg->size += len; | |
50 | ++qsg->nsg; | |
51 | } | |
52 | ||
53 | void qemu_sglist_destroy(QEMUSGList *qsg) | |
54 | { | |
f487b677 | 55 | object_unref(OBJECT(qsg->dev)); |
7267c094 | 56 | g_free(qsg->sg); |
ea8d82a1 | 57 | memset(qsg, 0, sizeof(*qsg)); |
244ab90e AL |
58 | } |
59 | ||
59a703eb | 60 | typedef struct { |
7c84b1b8 | 61 | BlockAIOCB common; |
8a8e63eb | 62 | AioContext *ctx; |
7c84b1b8 | 63 | BlockAIOCB *acb; |
59a703eb | 64 | QEMUSGList *sg; |
99868af3 | 65 | uint32_t align; |
d4f510eb | 66 | uint64_t offset; |
43cf8ae6 | 67 | DMADirection dir; |
59a703eb | 68 | int sg_cur_index; |
d3231181 | 69 | dma_addr_t sg_cur_byte; |
59a703eb AL |
70 | QEMUIOVector iov; |
71 | QEMUBH *bh; | |
cb144ccb | 72 | DMAIOFunc *io_func; |
8a8e63eb | 73 | void *io_func_opaque; |
37b7842c | 74 | } DMAAIOCB; |
59a703eb | 75 | |
4be74634 | 76 | static void dma_blk_cb(void *opaque, int ret); |
59a703eb AL |
77 | |
78 | static void reschedule_dma(void *opaque) | |
79 | { | |
37b7842c | 80 | DMAAIOCB *dbs = (DMAAIOCB *)opaque; |
59a703eb | 81 | |
539343c0 | 82 | assert(!dbs->acb && dbs->bh); |
59a703eb AL |
83 | qemu_bh_delete(dbs->bh); |
84 | dbs->bh = NULL; | |
4be74634 | 85 | dma_blk_cb(dbs, 0); |
59a703eb AL |
86 | } |
87 | ||
4be74634 | 88 | static void dma_blk_unmap(DMAAIOCB *dbs) |
59a703eb | 89 | { |
59a703eb AL |
90 | int i; |
91 | ||
59a703eb | 92 | for (i = 0; i < dbs->iov.niov; ++i) { |
df32fd1c | 93 | dma_memory_unmap(dbs->sg->as, dbs->iov.iov[i].iov_base, |
c65bcef3 DG |
94 | dbs->iov.iov[i].iov_len, dbs->dir, |
95 | dbs->iov.iov[i].iov_len); | |
59a703eb | 96 | } |
c3adb5b9 PB |
97 | qemu_iovec_reset(&dbs->iov); |
98 | } | |
99 | ||
100 | static void dma_complete(DMAAIOCB *dbs, int ret) | |
101 | { | |
c57c4658 KW |
102 | trace_dma_complete(dbs, ret, dbs->common.cb); |
103 | ||
539343c0 | 104 | assert(!dbs->acb && !dbs->bh); |
4be74634 | 105 | dma_blk_unmap(dbs); |
c3adb5b9 PB |
106 | if (dbs->common.cb) { |
107 | dbs->common.cb(dbs->common.opaque, ret); | |
108 | } | |
109 | qemu_iovec_destroy(&dbs->iov); | |
8007429a | 110 | qemu_aio_unref(dbs); |
7403b14e AL |
111 | } |
112 | ||
4be74634 | 113 | static void dma_blk_cb(void *opaque, int ret) |
7403b14e AL |
114 | { |
115 | DMAAIOCB *dbs = (DMAAIOCB *)opaque; | |
abfcd276 | 116 | AioContext *ctx = dbs->ctx; |
c65bcef3 | 117 | dma_addr_t cur_addr, cur_len; |
7403b14e AL |
118 | void *mem; |
119 | ||
4be74634 | 120 | trace_dma_blk_cb(dbs, ret); |
c57c4658 | 121 | |
abfcd276 | 122 | aio_context_acquire(ctx); |
7403b14e | 123 | dbs->acb = NULL; |
d4f510eb | 124 | dbs->offset += dbs->iov.size; |
59a703eb AL |
125 | |
126 | if (dbs->sg_cur_index == dbs->sg->nsg || ret < 0) { | |
c3adb5b9 | 127 | dma_complete(dbs, ret); |
abfcd276 | 128 | goto out; |
59a703eb | 129 | } |
4be74634 | 130 | dma_blk_unmap(dbs); |
59a703eb AL |
131 | |
132 | while (dbs->sg_cur_index < dbs->sg->nsg) { | |
133 | cur_addr = dbs->sg->sg[dbs->sg_cur_index].base + dbs->sg_cur_byte; | |
134 | cur_len = dbs->sg->sg[dbs->sg_cur_index].len - dbs->sg_cur_byte; | |
a1d4b0a3 PMD |
135 | mem = dma_memory_map(dbs->sg->as, cur_addr, &cur_len, dbs->dir, |
136 | MEMTXATTRS_UNSPECIFIED); | |
5fb0a6b5 PD |
137 | /* |
138 | * Make reads deterministic in icount mode. Windows sometimes issues | |
139 | * disk read requests with overlapping SGs. It leads | |
140 | * to non-determinism, because resulting buffer contents may be mixed | |
141 | * from several sectors. This code splits all SGs into several | |
142 | * groups. SGs in every group do not overlap. | |
143 | */ | |
740b1759 | 144 | if (mem && icount_enabled() && dbs->dir == DMA_DIRECTION_FROM_DEVICE) { |
5fb0a6b5 PD |
145 | int i; |
146 | for (i = 0 ; i < dbs->iov.niov ; ++i) { | |
147 | if (ranges_overlap((intptr_t)dbs->iov.iov[i].iov_base, | |
148 | dbs->iov.iov[i].iov_len, (intptr_t)mem, | |
149 | cur_len)) { | |
150 | dma_memory_unmap(dbs->sg->as, mem, cur_len, | |
151 | dbs->dir, cur_len); | |
152 | mem = NULL; | |
153 | break; | |
154 | } | |
155 | } | |
156 | } | |
59a703eb AL |
157 | if (!mem) |
158 | break; | |
159 | qemu_iovec_add(&dbs->iov, mem, cur_len); | |
160 | dbs->sg_cur_byte += cur_len; | |
161 | if (dbs->sg_cur_byte == dbs->sg->sg[dbs->sg_cur_index].len) { | |
162 | dbs->sg_cur_byte = 0; | |
163 | ++dbs->sg_cur_index; | |
164 | } | |
165 | } | |
166 | ||
167 | if (dbs->iov.size == 0) { | |
c57c4658 | 168 | trace_dma_map_wait(dbs); |
abfcd276 | 169 | dbs->bh = aio_bh_new(ctx, reschedule_dma, dbs); |
e95205e1 | 170 | cpu_register_map_client(dbs->bh); |
abfcd276 | 171 | goto out; |
59a703eb AL |
172 | } |
173 | ||
99868af3 MCA |
174 | if (!QEMU_IS_ALIGNED(dbs->iov.size, dbs->align)) { |
175 | qemu_iovec_discard_back(&dbs->iov, | |
176 | QEMU_ALIGN_DOWN(dbs->iov.size, dbs->align)); | |
58f423fb KW |
177 | } |
178 | ||
8a8e63eb PB |
179 | dbs->acb = dbs->io_func(dbs->offset, &dbs->iov, |
180 | dma_blk_cb, dbs, dbs->io_func_opaque); | |
6bee44ea | 181 | assert(dbs->acb); |
abfcd276 SH |
182 | out: |
183 | aio_context_release(ctx); | |
59a703eb AL |
184 | } |
185 | ||
7c84b1b8 | 186 | static void dma_aio_cancel(BlockAIOCB *acb) |
c16b5a2c CH |
187 | { |
188 | DMAAIOCB *dbs = container_of(acb, DMAAIOCB, common); | |
189 | ||
c57c4658 KW |
190 | trace_dma_aio_cancel(dbs); |
191 | ||
539343c0 | 192 | assert(!(dbs->acb && dbs->bh)); |
c16b5a2c | 193 | if (dbs->acb) { |
539343c0 | 194 | /* This will invoke dma_blk_cb. */ |
4be74634 | 195 | blk_aio_cancel_async(dbs->acb); |
539343c0 | 196 | return; |
c16b5a2c | 197 | } |
539343c0 | 198 | |
e95205e1 FZ |
199 | if (dbs->bh) { |
200 | cpu_unregister_map_client(dbs->bh); | |
201 | qemu_bh_delete(dbs->bh); | |
202 | dbs->bh = NULL; | |
203 | } | |
539343c0 PB |
204 | if (dbs->common.cb) { |
205 | dbs->common.cb(dbs->common.opaque, -ECANCELED); | |
206 | } | |
c16b5a2c CH |
207 | } |
208 | ||
d7331bed | 209 | static const AIOCBInfo dma_aiocb_info = { |
c16b5a2c | 210 | .aiocb_size = sizeof(DMAAIOCB), |
9bb9da46 | 211 | .cancel_async = dma_aio_cancel, |
c16b5a2c CH |
212 | }; |
213 | ||
8a8e63eb | 214 | BlockAIOCB *dma_blk_io(AioContext *ctx, |
99868af3 | 215 | QEMUSGList *sg, uint64_t offset, uint32_t align, |
8a8e63eb PB |
216 | DMAIOFunc *io_func, void *io_func_opaque, |
217 | BlockCompletionFunc *cb, | |
43cf8ae6 | 218 | void *opaque, DMADirection dir) |
59a703eb | 219 | { |
8a8e63eb | 220 | DMAAIOCB *dbs = qemu_aio_get(&dma_aiocb_info, NULL, cb, opaque); |
59a703eb | 221 | |
8a8e63eb | 222 | trace_dma_blk_io(dbs, io_func_opaque, offset, (dir == DMA_DIRECTION_TO_DEVICE)); |
c57c4658 | 223 | |
37b7842c | 224 | dbs->acb = NULL; |
59a703eb | 225 | dbs->sg = sg; |
8a8e63eb | 226 | dbs->ctx = ctx; |
cbe0ed62 | 227 | dbs->offset = offset; |
99868af3 | 228 | dbs->align = align; |
59a703eb AL |
229 | dbs->sg_cur_index = 0; |
230 | dbs->sg_cur_byte = 0; | |
43cf8ae6 | 231 | dbs->dir = dir; |
cb144ccb | 232 | dbs->io_func = io_func; |
8a8e63eb | 233 | dbs->io_func_opaque = io_func_opaque; |
59a703eb AL |
234 | dbs->bh = NULL; |
235 | qemu_iovec_init(&dbs->iov, sg->nsg); | |
4be74634 | 236 | dma_blk_cb(dbs, 0); |
37b7842c | 237 | return &dbs->common; |
59a703eb AL |
238 | } |
239 | ||
240 | ||
8a8e63eb PB |
241 | static |
242 | BlockAIOCB *dma_blk_read_io_func(int64_t offset, QEMUIOVector *iov, | |
243 | BlockCompletionFunc *cb, void *cb_opaque, | |
244 | void *opaque) | |
245 | { | |
246 | BlockBackend *blk = opaque; | |
247 | return blk_aio_preadv(blk, offset, iov, 0, cb, cb_opaque); | |
248 | } | |
249 | ||
4be74634 | 250 | BlockAIOCB *dma_blk_read(BlockBackend *blk, |
99868af3 | 251 | QEMUSGList *sg, uint64_t offset, uint32_t align, |
4be74634 | 252 | void (*cb)(void *opaque, int ret), void *opaque) |
59a703eb | 253 | { |
99868af3 MCA |
254 | return dma_blk_io(blk_get_aio_context(blk), sg, offset, align, |
255 | dma_blk_read_io_func, blk, cb, opaque, | |
4be74634 | 256 | DMA_DIRECTION_FROM_DEVICE); |
59a703eb AL |
257 | } |
258 | ||
8a8e63eb PB |
259 | static |
260 | BlockAIOCB *dma_blk_write_io_func(int64_t offset, QEMUIOVector *iov, | |
261 | BlockCompletionFunc *cb, void *cb_opaque, | |
262 | void *opaque) | |
263 | { | |
264 | BlockBackend *blk = opaque; | |
265 | return blk_aio_pwritev(blk, offset, iov, 0, cb, cb_opaque); | |
266 | } | |
267 | ||
4be74634 | 268 | BlockAIOCB *dma_blk_write(BlockBackend *blk, |
99868af3 | 269 | QEMUSGList *sg, uint64_t offset, uint32_t align, |
4be74634 | 270 | void (*cb)(void *opaque, int ret), void *opaque) |
59a703eb | 271 | { |
99868af3 MCA |
272 | return dma_blk_io(blk_get_aio_context(blk), sg, offset, align, |
273 | dma_blk_write_io_func, blk, cb, opaque, | |
4be74634 | 274 | DMA_DIRECTION_TO_DEVICE); |
59a703eb | 275 | } |
8171ee35 PB |
276 | |
277 | ||
bfa30f39 | 278 | static MemTxResult dma_buf_rw(void *buf, dma_addr_t len, dma_addr_t *residual, |
292e1314 PMD |
279 | QEMUSGList *sg, DMADirection dir, |
280 | MemTxAttrs attrs) | |
8171ee35 | 281 | { |
c0ee1527 | 282 | uint8_t *ptr = buf; |
bfa30f39 | 283 | dma_addr_t xresidual; |
8171ee35 | 284 | int sg_cur_index; |
292e1314 | 285 | MemTxResult res = MEMTX_OK; |
8171ee35 | 286 | |
5f412602 | 287 | xresidual = sg->size; |
8171ee35 | 288 | sg_cur_index = 0; |
5f412602 | 289 | len = MIN(len, xresidual); |
8171ee35 PB |
290 | while (len > 0) { |
291 | ScatterGatherEntry entry = sg->sg[sg_cur_index++]; | |
bfa30f39 | 292 | dma_addr_t xfer = MIN(len, entry.len); |
292e1314 | 293 | res |= dma_memory_rw(sg->as, entry.base, ptr, xfer, dir, attrs); |
8171ee35 PB |
294 | ptr += xfer; |
295 | len -= xfer; | |
5f412602 | 296 | xresidual -= xfer; |
8171ee35 PB |
297 | } |
298 | ||
5f412602 PMD |
299 | if (residual) { |
300 | *residual = xresidual; | |
292e1314 PMD |
301 | } |
302 | return res; | |
8171ee35 PB |
303 | } |
304 | ||
f02b664a | 305 | MemTxResult dma_buf_read(void *ptr, dma_addr_t len, dma_addr_t *residual, |
bfa30f39 | 306 | QEMUSGList *sg, MemTxAttrs attrs) |
8171ee35 | 307 | { |
f02b664a | 308 | return dma_buf_rw(ptr, len, residual, sg, DMA_DIRECTION_FROM_DEVICE, attrs); |
8171ee35 PB |
309 | } |
310 | ||
f02b664a | 311 | MemTxResult dma_buf_write(void *ptr, dma_addr_t len, dma_addr_t *residual, |
bfa30f39 | 312 | QEMUSGList *sg, MemTxAttrs attrs) |
8171ee35 | 313 | { |
f02b664a | 314 | return dma_buf_rw(ptr, len, residual, sg, DMA_DIRECTION_TO_DEVICE, attrs); |
8171ee35 | 315 | } |
84a69356 | 316 | |
4be74634 | 317 | void dma_acct_start(BlockBackend *blk, BlockAcctCookie *cookie, |
84a69356 PB |
318 | QEMUSGList *sg, enum BlockAcctType type) |
319 | { | |
4be74634 | 320 | block_acct_start(blk_get_stats(blk), cookie, sg->size, type); |
84a69356 | 321 | } |
f14fb6c2 EA |
322 | |
323 | uint64_t dma_aligned_pow2_mask(uint64_t start, uint64_t end, int max_addr_bits) | |
324 | { | |
325 | uint64_t max_mask = UINT64_MAX, addr_mask = end - start; | |
326 | uint64_t alignment_mask, size_mask; | |
327 | ||
328 | if (max_addr_bits != 64) { | |
329 | max_mask = (1ULL << max_addr_bits) - 1; | |
330 | } | |
331 | ||
332 | alignment_mask = start ? (start & -start) - 1 : max_mask; | |
333 | alignment_mask = MIN(alignment_mask, max_mask); | |
334 | size_mask = MIN(addr_mask, max_mask); | |
335 | ||
336 | if (alignment_mask <= size_mask) { | |
337 | /* Increase the alignment of start */ | |
338 | return alignment_mask; | |
339 | } else { | |
340 | /* Find the largest page mask from size */ | |
341 | if (addr_mask == UINT64_MAX) { | |
342 | return UINT64_MAX; | |
343 | } | |
344 | return (1ULL << (63 - clz64(addr_mask + 1))) - 1; | |
345 | } | |
346 | } | |
347 |