]>
Commit | Line | Data |
---|---|---|
244ab90e AL |
1 | /* |
2 | * DMA helper functions | |
3 | * | |
bb755f52 | 4 | * Copyright (c) 2009,2020 Red Hat |
244ab90e AL |
5 | * |
6 | * This work is licensed under the terms of the GNU General Public License | |
7 | * (GNU GPL), version 2 or later. | |
8 | */ | |
9 | ||
d38ea87a | 10 | #include "qemu/osdep.h" |
4be74634 | 11 | #include "sysemu/block-backend.h" |
9c17d615 | 12 | #include "sysemu/dma.h" |
243af022 | 13 | #include "trace/trace-root.h" |
1de7afc9 | 14 | #include "qemu/thread.h" |
6a1751b7 | 15 | #include "qemu/main-loop.h" |
740b1759 | 16 | #include "sysemu/cpu-timers.h" |
5fb0a6b5 | 17 | #include "qemu/range.h" |
244ab90e | 18 | |
e5332e63 DG |
19 | /* #define DEBUG_IOMMU */ |
20 | ||
bb755f52 | 21 | MemTxResult dma_memory_set(AddressSpace *as, dma_addr_t addr, |
7a36e42d | 22 | uint8_t c, dma_addr_t len, MemTxAttrs attrs) |
d86a77f8 | 23 | { |
df32fd1c | 24 | dma_barrier(as, DMA_DIRECTION_FROM_DEVICE); |
24addbc7 | 25 | |
75f01c68 | 26 | return address_space_set(as, addr, c, len, attrs); |
d86a77f8 DG |
27 | } |
28 | ||
f487b677 PB |
29 | void qemu_sglist_init(QEMUSGList *qsg, DeviceState *dev, int alloc_hint, |
30 | AddressSpace *as) | |
244ab90e | 31 | { |
b21e2380 | 32 | qsg->sg = g_new(ScatterGatherEntry, alloc_hint); |
244ab90e AL |
33 | qsg->nsg = 0; |
34 | qsg->nalloc = alloc_hint; | |
35 | qsg->size = 0; | |
df32fd1c | 36 | qsg->as = as; |
f487b677 PB |
37 | qsg->dev = dev; |
38 | object_ref(OBJECT(dev)); | |
244ab90e AL |
39 | } |
40 | ||
d3231181 | 41 | void qemu_sglist_add(QEMUSGList *qsg, dma_addr_t base, dma_addr_t len) |
244ab90e AL |
42 | { |
43 | if (qsg->nsg == qsg->nalloc) { | |
44 | qsg->nalloc = 2 * qsg->nalloc + 1; | |
b21e2380 | 45 | qsg->sg = g_renew(ScatterGatherEntry, qsg->sg, qsg->nalloc); |
244ab90e AL |
46 | } |
47 | qsg->sg[qsg->nsg].base = base; | |
48 | qsg->sg[qsg->nsg].len = len; | |
49 | qsg->size += len; | |
50 | ++qsg->nsg; | |
51 | } | |
52 | ||
53 | void qemu_sglist_destroy(QEMUSGList *qsg) | |
54 | { | |
f487b677 | 55 | object_unref(OBJECT(qsg->dev)); |
7267c094 | 56 | g_free(qsg->sg); |
ea8d82a1 | 57 | memset(qsg, 0, sizeof(*qsg)); |
244ab90e AL |
58 | } |
59 | ||
59a703eb | 60 | typedef struct { |
7c84b1b8 | 61 | BlockAIOCB common; |
8a8e63eb | 62 | AioContext *ctx; |
7c84b1b8 | 63 | BlockAIOCB *acb; |
59a703eb | 64 | QEMUSGList *sg; |
99868af3 | 65 | uint32_t align; |
d4f510eb | 66 | uint64_t offset; |
43cf8ae6 | 67 | DMADirection dir; |
59a703eb | 68 | int sg_cur_index; |
d3231181 | 69 | dma_addr_t sg_cur_byte; |
59a703eb AL |
70 | QEMUIOVector iov; |
71 | QEMUBH *bh; | |
cb144ccb | 72 | DMAIOFunc *io_func; |
8a8e63eb | 73 | void *io_func_opaque; |
37b7842c | 74 | } DMAAIOCB; |
59a703eb | 75 | |
4be74634 | 76 | static void dma_blk_cb(void *opaque, int ret); |
59a703eb AL |
77 | |
78 | static void reschedule_dma(void *opaque) | |
79 | { | |
37b7842c | 80 | DMAAIOCB *dbs = (DMAAIOCB *)opaque; |
59a703eb | 81 | |
539343c0 | 82 | assert(!dbs->acb && dbs->bh); |
59a703eb AL |
83 | qemu_bh_delete(dbs->bh); |
84 | dbs->bh = NULL; | |
4be74634 | 85 | dma_blk_cb(dbs, 0); |
59a703eb AL |
86 | } |
87 | ||
4be74634 | 88 | static void dma_blk_unmap(DMAAIOCB *dbs) |
59a703eb | 89 | { |
59a703eb AL |
90 | int i; |
91 | ||
59a703eb | 92 | for (i = 0; i < dbs->iov.niov; ++i) { |
df32fd1c | 93 | dma_memory_unmap(dbs->sg->as, dbs->iov.iov[i].iov_base, |
c65bcef3 DG |
94 | dbs->iov.iov[i].iov_len, dbs->dir, |
95 | dbs->iov.iov[i].iov_len); | |
59a703eb | 96 | } |
c3adb5b9 PB |
97 | qemu_iovec_reset(&dbs->iov); |
98 | } | |
99 | ||
100 | static void dma_complete(DMAAIOCB *dbs, int ret) | |
101 | { | |
c57c4658 KW |
102 | trace_dma_complete(dbs, ret, dbs->common.cb); |
103 | ||
539343c0 | 104 | assert(!dbs->acb && !dbs->bh); |
4be74634 | 105 | dma_blk_unmap(dbs); |
c3adb5b9 PB |
106 | if (dbs->common.cb) { |
107 | dbs->common.cb(dbs->common.opaque, ret); | |
108 | } | |
109 | qemu_iovec_destroy(&dbs->iov); | |
8007429a | 110 | qemu_aio_unref(dbs); |
7403b14e AL |
111 | } |
112 | ||
4be74634 | 113 | static void dma_blk_cb(void *opaque, int ret) |
7403b14e AL |
114 | { |
115 | DMAAIOCB *dbs = (DMAAIOCB *)opaque; | |
abfcd276 | 116 | AioContext *ctx = dbs->ctx; |
c65bcef3 | 117 | dma_addr_t cur_addr, cur_len; |
7403b14e AL |
118 | void *mem; |
119 | ||
4be74634 | 120 | trace_dma_blk_cb(dbs, ret); |
c57c4658 | 121 | |
abfcd276 | 122 | aio_context_acquire(ctx); |
7403b14e | 123 | dbs->acb = NULL; |
d4f510eb | 124 | dbs->offset += dbs->iov.size; |
59a703eb AL |
125 | |
126 | if (dbs->sg_cur_index == dbs->sg->nsg || ret < 0) { | |
c3adb5b9 | 127 | dma_complete(dbs, ret); |
abfcd276 | 128 | goto out; |
59a703eb | 129 | } |
4be74634 | 130 | dma_blk_unmap(dbs); |
59a703eb AL |
131 | |
132 | while (dbs->sg_cur_index < dbs->sg->nsg) { | |
133 | cur_addr = dbs->sg->sg[dbs->sg_cur_index].base + dbs->sg_cur_byte; | |
134 | cur_len = dbs->sg->sg[dbs->sg_cur_index].len - dbs->sg_cur_byte; | |
a1d4b0a3 PMD |
135 | mem = dma_memory_map(dbs->sg->as, cur_addr, &cur_len, dbs->dir, |
136 | MEMTXATTRS_UNSPECIFIED); | |
5fb0a6b5 PD |
137 | /* |
138 | * Make reads deterministic in icount mode. Windows sometimes issues | |
139 | * disk read requests with overlapping SGs. It leads | |
140 | * to non-determinism, because resulting buffer contents may be mixed | |
141 | * from several sectors. This code splits all SGs into several | |
142 | * groups. SGs in every group do not overlap. | |
143 | */ | |
740b1759 | 144 | if (mem && icount_enabled() && dbs->dir == DMA_DIRECTION_FROM_DEVICE) { |
5fb0a6b5 PD |
145 | int i; |
146 | for (i = 0 ; i < dbs->iov.niov ; ++i) { | |
147 | if (ranges_overlap((intptr_t)dbs->iov.iov[i].iov_base, | |
148 | dbs->iov.iov[i].iov_len, (intptr_t)mem, | |
149 | cur_len)) { | |
150 | dma_memory_unmap(dbs->sg->as, mem, cur_len, | |
151 | dbs->dir, cur_len); | |
152 | mem = NULL; | |
153 | break; | |
154 | } | |
155 | } | |
156 | } | |
59a703eb AL |
157 | if (!mem) |
158 | break; | |
159 | qemu_iovec_add(&dbs->iov, mem, cur_len); | |
160 | dbs->sg_cur_byte += cur_len; | |
161 | if (dbs->sg_cur_byte == dbs->sg->sg[dbs->sg_cur_index].len) { | |
162 | dbs->sg_cur_byte = 0; | |
163 | ++dbs->sg_cur_index; | |
164 | } | |
165 | } | |
166 | ||
167 | if (dbs->iov.size == 0) { | |
c57c4658 | 168 | trace_dma_map_wait(dbs); |
abfcd276 | 169 | dbs->bh = aio_bh_new(ctx, reschedule_dma, dbs); |
e95205e1 | 170 | cpu_register_map_client(dbs->bh); |
abfcd276 | 171 | goto out; |
59a703eb AL |
172 | } |
173 | ||
99868af3 MCA |
174 | if (!QEMU_IS_ALIGNED(dbs->iov.size, dbs->align)) { |
175 | qemu_iovec_discard_back(&dbs->iov, | |
176 | QEMU_ALIGN_DOWN(dbs->iov.size, dbs->align)); | |
58f423fb KW |
177 | } |
178 | ||
8a8e63eb PB |
179 | dbs->acb = dbs->io_func(dbs->offset, &dbs->iov, |
180 | dma_blk_cb, dbs, dbs->io_func_opaque); | |
6bee44ea | 181 | assert(dbs->acb); |
abfcd276 SH |
182 | out: |
183 | aio_context_release(ctx); | |
59a703eb AL |
184 | } |
185 | ||
7c84b1b8 | 186 | static void dma_aio_cancel(BlockAIOCB *acb) |
c16b5a2c CH |
187 | { |
188 | DMAAIOCB *dbs = container_of(acb, DMAAIOCB, common); | |
189 | ||
c57c4658 KW |
190 | trace_dma_aio_cancel(dbs); |
191 | ||
539343c0 | 192 | assert(!(dbs->acb && dbs->bh)); |
c16b5a2c | 193 | if (dbs->acb) { |
539343c0 | 194 | /* This will invoke dma_blk_cb. */ |
4be74634 | 195 | blk_aio_cancel_async(dbs->acb); |
539343c0 | 196 | return; |
c16b5a2c | 197 | } |
539343c0 | 198 | |
e95205e1 FZ |
199 | if (dbs->bh) { |
200 | cpu_unregister_map_client(dbs->bh); | |
201 | qemu_bh_delete(dbs->bh); | |
202 | dbs->bh = NULL; | |
203 | } | |
539343c0 PB |
204 | if (dbs->common.cb) { |
205 | dbs->common.cb(dbs->common.opaque, -ECANCELED); | |
206 | } | |
c16b5a2c CH |
207 | } |
208 | ||
5fa78b2a SH |
209 | static AioContext *dma_get_aio_context(BlockAIOCB *acb) |
210 | { | |
211 | DMAAIOCB *dbs = container_of(acb, DMAAIOCB, common); | |
212 | ||
213 | return dbs->ctx; | |
214 | } | |
9bb9da46 | 215 | |
d7331bed | 216 | static const AIOCBInfo dma_aiocb_info = { |
c16b5a2c | 217 | .aiocb_size = sizeof(DMAAIOCB), |
9bb9da46 | 218 | .cancel_async = dma_aio_cancel, |
5fa78b2a | 219 | .get_aio_context = dma_get_aio_context, |
c16b5a2c CH |
220 | }; |
221 | ||
8a8e63eb | 222 | BlockAIOCB *dma_blk_io(AioContext *ctx, |
99868af3 | 223 | QEMUSGList *sg, uint64_t offset, uint32_t align, |
8a8e63eb PB |
224 | DMAIOFunc *io_func, void *io_func_opaque, |
225 | BlockCompletionFunc *cb, | |
43cf8ae6 | 226 | void *opaque, DMADirection dir) |
59a703eb | 227 | { |
8a8e63eb | 228 | DMAAIOCB *dbs = qemu_aio_get(&dma_aiocb_info, NULL, cb, opaque); |
59a703eb | 229 | |
8a8e63eb | 230 | trace_dma_blk_io(dbs, io_func_opaque, offset, (dir == DMA_DIRECTION_TO_DEVICE)); |
c57c4658 | 231 | |
37b7842c | 232 | dbs->acb = NULL; |
59a703eb | 233 | dbs->sg = sg; |
8a8e63eb | 234 | dbs->ctx = ctx; |
cbe0ed62 | 235 | dbs->offset = offset; |
99868af3 | 236 | dbs->align = align; |
59a703eb AL |
237 | dbs->sg_cur_index = 0; |
238 | dbs->sg_cur_byte = 0; | |
43cf8ae6 | 239 | dbs->dir = dir; |
cb144ccb | 240 | dbs->io_func = io_func; |
8a8e63eb | 241 | dbs->io_func_opaque = io_func_opaque; |
59a703eb AL |
242 | dbs->bh = NULL; |
243 | qemu_iovec_init(&dbs->iov, sg->nsg); | |
4be74634 | 244 | dma_blk_cb(dbs, 0); |
37b7842c | 245 | return &dbs->common; |
59a703eb AL |
246 | } |
247 | ||
248 | ||
8a8e63eb PB |
249 | static |
250 | BlockAIOCB *dma_blk_read_io_func(int64_t offset, QEMUIOVector *iov, | |
251 | BlockCompletionFunc *cb, void *cb_opaque, | |
252 | void *opaque) | |
253 | { | |
254 | BlockBackend *blk = opaque; | |
255 | return blk_aio_preadv(blk, offset, iov, 0, cb, cb_opaque); | |
256 | } | |
257 | ||
4be74634 | 258 | BlockAIOCB *dma_blk_read(BlockBackend *blk, |
99868af3 | 259 | QEMUSGList *sg, uint64_t offset, uint32_t align, |
4be74634 | 260 | void (*cb)(void *opaque, int ret), void *opaque) |
59a703eb | 261 | { |
99868af3 MCA |
262 | return dma_blk_io(blk_get_aio_context(blk), sg, offset, align, |
263 | dma_blk_read_io_func, blk, cb, opaque, | |
4be74634 | 264 | DMA_DIRECTION_FROM_DEVICE); |
59a703eb AL |
265 | } |
266 | ||
8a8e63eb PB |
267 | static |
268 | BlockAIOCB *dma_blk_write_io_func(int64_t offset, QEMUIOVector *iov, | |
269 | BlockCompletionFunc *cb, void *cb_opaque, | |
270 | void *opaque) | |
271 | { | |
272 | BlockBackend *blk = opaque; | |
273 | return blk_aio_pwritev(blk, offset, iov, 0, cb, cb_opaque); | |
274 | } | |
275 | ||
4be74634 | 276 | BlockAIOCB *dma_blk_write(BlockBackend *blk, |
99868af3 | 277 | QEMUSGList *sg, uint64_t offset, uint32_t align, |
4be74634 | 278 | void (*cb)(void *opaque, int ret), void *opaque) |
59a703eb | 279 | { |
99868af3 MCA |
280 | return dma_blk_io(blk_get_aio_context(blk), sg, offset, align, |
281 | dma_blk_write_io_func, blk, cb, opaque, | |
4be74634 | 282 | DMA_DIRECTION_TO_DEVICE); |
59a703eb | 283 | } |
8171ee35 PB |
284 | |
285 | ||
bfa30f39 | 286 | static MemTxResult dma_buf_rw(void *buf, dma_addr_t len, dma_addr_t *residual, |
292e1314 PMD |
287 | QEMUSGList *sg, DMADirection dir, |
288 | MemTxAttrs attrs) | |
8171ee35 | 289 | { |
c0ee1527 | 290 | uint8_t *ptr = buf; |
bfa30f39 | 291 | dma_addr_t xresidual; |
8171ee35 | 292 | int sg_cur_index; |
292e1314 | 293 | MemTxResult res = MEMTX_OK; |
8171ee35 | 294 | |
5f412602 | 295 | xresidual = sg->size; |
8171ee35 | 296 | sg_cur_index = 0; |
5f412602 | 297 | len = MIN(len, xresidual); |
8171ee35 PB |
298 | while (len > 0) { |
299 | ScatterGatherEntry entry = sg->sg[sg_cur_index++]; | |
bfa30f39 | 300 | dma_addr_t xfer = MIN(len, entry.len); |
292e1314 | 301 | res |= dma_memory_rw(sg->as, entry.base, ptr, xfer, dir, attrs); |
8171ee35 PB |
302 | ptr += xfer; |
303 | len -= xfer; | |
5f412602 | 304 | xresidual -= xfer; |
8171ee35 PB |
305 | } |
306 | ||
5f412602 PMD |
307 | if (residual) { |
308 | *residual = xresidual; | |
292e1314 PMD |
309 | } |
310 | return res; | |
8171ee35 PB |
311 | } |
312 | ||
f02b664a | 313 | MemTxResult dma_buf_read(void *ptr, dma_addr_t len, dma_addr_t *residual, |
bfa30f39 | 314 | QEMUSGList *sg, MemTxAttrs attrs) |
8171ee35 | 315 | { |
f02b664a | 316 | return dma_buf_rw(ptr, len, residual, sg, DMA_DIRECTION_FROM_DEVICE, attrs); |
8171ee35 PB |
317 | } |
318 | ||
f02b664a | 319 | MemTxResult dma_buf_write(void *ptr, dma_addr_t len, dma_addr_t *residual, |
bfa30f39 | 320 | QEMUSGList *sg, MemTxAttrs attrs) |
8171ee35 | 321 | { |
f02b664a | 322 | return dma_buf_rw(ptr, len, residual, sg, DMA_DIRECTION_TO_DEVICE, attrs); |
8171ee35 | 323 | } |
84a69356 | 324 | |
4be74634 | 325 | void dma_acct_start(BlockBackend *blk, BlockAcctCookie *cookie, |
84a69356 PB |
326 | QEMUSGList *sg, enum BlockAcctType type) |
327 | { | |
4be74634 | 328 | block_acct_start(blk_get_stats(blk), cookie, sg->size, type); |
84a69356 | 329 | } |
f14fb6c2 EA |
330 | |
331 | uint64_t dma_aligned_pow2_mask(uint64_t start, uint64_t end, int max_addr_bits) | |
332 | { | |
333 | uint64_t max_mask = UINT64_MAX, addr_mask = end - start; | |
334 | uint64_t alignment_mask, size_mask; | |
335 | ||
336 | if (max_addr_bits != 64) { | |
337 | max_mask = (1ULL << max_addr_bits) - 1; | |
338 | } | |
339 | ||
340 | alignment_mask = start ? (start & -start) - 1 : max_mask; | |
341 | alignment_mask = MIN(alignment_mask, max_mask); | |
342 | size_mask = MIN(addr_mask, max_mask); | |
343 | ||
344 | if (alignment_mask <= size_mask) { | |
345 | /* Increase the alignment of start */ | |
346 | return alignment_mask; | |
347 | } else { | |
348 | /* Find the largest page mask from size */ | |
349 | if (addr_mask == UINT64_MAX) { | |
350 | return UINT64_MAX; | |
351 | } | |
352 | return (1ULL << (63 - clz64(addr_mask + 1))) - 1; | |
353 | } | |
354 | } | |
355 |