]>
Commit | Line | Data |
---|---|---|
244ab90e AL |
1 | /* |
2 | * DMA helper functions | |
3 | * | |
4 | * Copyright (c) 2009 Red Hat | |
5 | * | |
6 | * This work is licensed under the terms of the GNU General Public License | |
7 | * (GNU GPL), version 2 or later. | |
8 | */ | |
9 | ||
9c17d615 | 10 | #include "sysemu/dma.h" |
c57c4658 | 11 | #include "trace.h" |
1de7afc9 PB |
12 | #include "qemu/range.h" |
13 | #include "qemu/thread.h" | |
244ab90e | 14 | |
e5332e63 DG |
15 | /* #define DEBUG_IOMMU */ |
16 | ||
b90600ee AK |
17 | static void do_dma_memory_set(AddressSpace *as, |
18 | dma_addr_t addr, uint8_t c, dma_addr_t len) | |
d86a77f8 DG |
19 | { |
20 | #define FILLBUF_SIZE 512 | |
21 | uint8_t fillbuf[FILLBUF_SIZE]; | |
22 | int l; | |
23 | ||
24 | memset(fillbuf, c, FILLBUF_SIZE); | |
25 | while (len > 0) { | |
26 | l = len < FILLBUF_SIZE ? len : FILLBUF_SIZE; | |
b90600ee | 27 | address_space_rw(as, addr, fillbuf, l, true); |
bc9b78de BH |
28 | len -= l; |
29 | addr += l; | |
d86a77f8 | 30 | } |
e5332e63 DG |
31 | } |
32 | ||
33 | int dma_memory_set(DMAContext *dma, dma_addr_t addr, uint8_t c, dma_addr_t len) | |
34 | { | |
7a0bac4d BH |
35 | dma_barrier(dma, DMA_DIRECTION_FROM_DEVICE); |
36 | ||
e5332e63 DG |
37 | if (dma_has_iommu(dma)) { |
38 | return iommu_dma_memory_set(dma, addr, c, len); | |
39 | } | |
b90600ee | 40 | do_dma_memory_set(dma->as, addr, c, len); |
e5332e63 | 41 | |
d86a77f8 DG |
42 | return 0; |
43 | } | |
44 | ||
c65bcef3 | 45 | void qemu_sglist_init(QEMUSGList *qsg, int alloc_hint, DMAContext *dma) |
244ab90e | 46 | { |
7267c094 | 47 | qsg->sg = g_malloc(alloc_hint * sizeof(ScatterGatherEntry)); |
244ab90e AL |
48 | qsg->nsg = 0; |
49 | qsg->nalloc = alloc_hint; | |
50 | qsg->size = 0; | |
c65bcef3 | 51 | qsg->dma = dma; |
244ab90e AL |
52 | } |
53 | ||
d3231181 | 54 | void qemu_sglist_add(QEMUSGList *qsg, dma_addr_t base, dma_addr_t len) |
244ab90e AL |
55 | { |
56 | if (qsg->nsg == qsg->nalloc) { | |
57 | qsg->nalloc = 2 * qsg->nalloc + 1; | |
7267c094 | 58 | qsg->sg = g_realloc(qsg->sg, qsg->nalloc * sizeof(ScatterGatherEntry)); |
244ab90e AL |
59 | } |
60 | qsg->sg[qsg->nsg].base = base; | |
61 | qsg->sg[qsg->nsg].len = len; | |
62 | qsg->size += len; | |
63 | ++qsg->nsg; | |
64 | } | |
65 | ||
66 | void qemu_sglist_destroy(QEMUSGList *qsg) | |
67 | { | |
7267c094 | 68 | g_free(qsg->sg); |
ea8d82a1 | 69 | memset(qsg, 0, sizeof(*qsg)); |
244ab90e AL |
70 | } |
71 | ||
59a703eb | 72 | typedef struct { |
37b7842c | 73 | BlockDriverAIOCB common; |
59a703eb AL |
74 | BlockDriverState *bs; |
75 | BlockDriverAIOCB *acb; | |
76 | QEMUSGList *sg; | |
77 | uint64_t sector_num; | |
43cf8ae6 | 78 | DMADirection dir; |
c3adb5b9 | 79 | bool in_cancel; |
59a703eb | 80 | int sg_cur_index; |
d3231181 | 81 | dma_addr_t sg_cur_byte; |
59a703eb AL |
82 | QEMUIOVector iov; |
83 | QEMUBH *bh; | |
cb144ccb | 84 | DMAIOFunc *io_func; |
37b7842c | 85 | } DMAAIOCB; |
59a703eb AL |
86 | |
87 | static void dma_bdrv_cb(void *opaque, int ret); | |
88 | ||
89 | static void reschedule_dma(void *opaque) | |
90 | { | |
37b7842c | 91 | DMAAIOCB *dbs = (DMAAIOCB *)opaque; |
59a703eb AL |
92 | |
93 | qemu_bh_delete(dbs->bh); | |
94 | dbs->bh = NULL; | |
c3adb5b9 | 95 | dma_bdrv_cb(dbs, 0); |
59a703eb AL |
96 | } |
97 | ||
98 | static void continue_after_map_failure(void *opaque) | |
99 | { | |
37b7842c | 100 | DMAAIOCB *dbs = (DMAAIOCB *)opaque; |
59a703eb AL |
101 | |
102 | dbs->bh = qemu_bh_new(reschedule_dma, dbs); | |
103 | qemu_bh_schedule(dbs->bh); | |
104 | } | |
105 | ||
7403b14e | 106 | static void dma_bdrv_unmap(DMAAIOCB *dbs) |
59a703eb | 107 | { |
59a703eb AL |
108 | int i; |
109 | ||
59a703eb | 110 | for (i = 0; i < dbs->iov.niov; ++i) { |
c65bcef3 DG |
111 | dma_memory_unmap(dbs->sg->dma, dbs->iov.iov[i].iov_base, |
112 | dbs->iov.iov[i].iov_len, dbs->dir, | |
113 | dbs->iov.iov[i].iov_len); | |
59a703eb | 114 | } |
c3adb5b9 PB |
115 | qemu_iovec_reset(&dbs->iov); |
116 | } | |
117 | ||
118 | static void dma_complete(DMAAIOCB *dbs, int ret) | |
119 | { | |
c57c4658 KW |
120 | trace_dma_complete(dbs, ret, dbs->common.cb); |
121 | ||
c3adb5b9 PB |
122 | dma_bdrv_unmap(dbs); |
123 | if (dbs->common.cb) { | |
124 | dbs->common.cb(dbs->common.opaque, ret); | |
125 | } | |
126 | qemu_iovec_destroy(&dbs->iov); | |
127 | if (dbs->bh) { | |
128 | qemu_bh_delete(dbs->bh); | |
129 | dbs->bh = NULL; | |
130 | } | |
131 | if (!dbs->in_cancel) { | |
132 | /* Requests may complete while dma_aio_cancel is in progress. In | |
133 | * this case, the AIOCB should not be released because it is still | |
134 | * referenced by dma_aio_cancel. */ | |
135 | qemu_aio_release(dbs); | |
136 | } | |
7403b14e AL |
137 | } |
138 | ||
856ae5c3 | 139 | static void dma_bdrv_cb(void *opaque, int ret) |
7403b14e AL |
140 | { |
141 | DMAAIOCB *dbs = (DMAAIOCB *)opaque; | |
c65bcef3 | 142 | dma_addr_t cur_addr, cur_len; |
7403b14e AL |
143 | void *mem; |
144 | ||
c57c4658 KW |
145 | trace_dma_bdrv_cb(dbs, ret); |
146 | ||
7403b14e AL |
147 | dbs->acb = NULL; |
148 | dbs->sector_num += dbs->iov.size / 512; | |
149 | dma_bdrv_unmap(dbs); | |
59a703eb AL |
150 | |
151 | if (dbs->sg_cur_index == dbs->sg->nsg || ret < 0) { | |
c3adb5b9 | 152 | dma_complete(dbs, ret); |
59a703eb AL |
153 | return; |
154 | } | |
155 | ||
156 | while (dbs->sg_cur_index < dbs->sg->nsg) { | |
157 | cur_addr = dbs->sg->sg[dbs->sg_cur_index].base + dbs->sg_cur_byte; | |
158 | cur_len = dbs->sg->sg[dbs->sg_cur_index].len - dbs->sg_cur_byte; | |
c65bcef3 | 159 | mem = dma_memory_map(dbs->sg->dma, cur_addr, &cur_len, dbs->dir); |
59a703eb AL |
160 | if (!mem) |
161 | break; | |
162 | qemu_iovec_add(&dbs->iov, mem, cur_len); | |
163 | dbs->sg_cur_byte += cur_len; | |
164 | if (dbs->sg_cur_byte == dbs->sg->sg[dbs->sg_cur_index].len) { | |
165 | dbs->sg_cur_byte = 0; | |
166 | ++dbs->sg_cur_index; | |
167 | } | |
168 | } | |
169 | ||
170 | if (dbs->iov.size == 0) { | |
c57c4658 | 171 | trace_dma_map_wait(dbs); |
59a703eb AL |
172 | cpu_register_map_client(dbs, continue_after_map_failure); |
173 | return; | |
174 | } | |
175 | ||
cb144ccb CH |
176 | dbs->acb = dbs->io_func(dbs->bs, dbs->sector_num, &dbs->iov, |
177 | dbs->iov.size / 512, dma_bdrv_cb, dbs); | |
6bee44ea | 178 | assert(dbs->acb); |
59a703eb AL |
179 | } |
180 | ||
c16b5a2c CH |
181 | static void dma_aio_cancel(BlockDriverAIOCB *acb) |
182 | { | |
183 | DMAAIOCB *dbs = container_of(acb, DMAAIOCB, common); | |
184 | ||
c57c4658 KW |
185 | trace_dma_aio_cancel(dbs); |
186 | ||
c16b5a2c | 187 | if (dbs->acb) { |
c3adb5b9 PB |
188 | BlockDriverAIOCB *acb = dbs->acb; |
189 | dbs->acb = NULL; | |
190 | dbs->in_cancel = true; | |
191 | bdrv_aio_cancel(acb); | |
192 | dbs->in_cancel = false; | |
c16b5a2c | 193 | } |
c3adb5b9 PB |
194 | dbs->common.cb = NULL; |
195 | dma_complete(dbs, 0); | |
c16b5a2c CH |
196 | } |
197 | ||
d7331bed | 198 | static const AIOCBInfo dma_aiocb_info = { |
c16b5a2c CH |
199 | .aiocb_size = sizeof(DMAAIOCB), |
200 | .cancel = dma_aio_cancel, | |
201 | }; | |
202 | ||
cb144ccb | 203 | BlockDriverAIOCB *dma_bdrv_io( |
59a703eb | 204 | BlockDriverState *bs, QEMUSGList *sg, uint64_t sector_num, |
cb144ccb | 205 | DMAIOFunc *io_func, BlockDriverCompletionFunc *cb, |
43cf8ae6 | 206 | void *opaque, DMADirection dir) |
59a703eb | 207 | { |
d7331bed | 208 | DMAAIOCB *dbs = qemu_aio_get(&dma_aiocb_info, bs, cb, opaque); |
59a703eb | 209 | |
43cf8ae6 | 210 | trace_dma_bdrv_io(dbs, bs, sector_num, (dir == DMA_DIRECTION_TO_DEVICE)); |
c57c4658 | 211 | |
37b7842c | 212 | dbs->acb = NULL; |
59a703eb | 213 | dbs->bs = bs; |
59a703eb AL |
214 | dbs->sg = sg; |
215 | dbs->sector_num = sector_num; | |
216 | dbs->sg_cur_index = 0; | |
217 | dbs->sg_cur_byte = 0; | |
43cf8ae6 | 218 | dbs->dir = dir; |
cb144ccb | 219 | dbs->io_func = io_func; |
59a703eb AL |
220 | dbs->bh = NULL; |
221 | qemu_iovec_init(&dbs->iov, sg->nsg); | |
222 | dma_bdrv_cb(dbs, 0); | |
37b7842c | 223 | return &dbs->common; |
59a703eb AL |
224 | } |
225 | ||
226 | ||
227 | BlockDriverAIOCB *dma_bdrv_read(BlockDriverState *bs, | |
228 | QEMUSGList *sg, uint64_t sector, | |
229 | void (*cb)(void *opaque, int ret), void *opaque) | |
230 | { | |
43cf8ae6 DG |
231 | return dma_bdrv_io(bs, sg, sector, bdrv_aio_readv, cb, opaque, |
232 | DMA_DIRECTION_FROM_DEVICE); | |
59a703eb AL |
233 | } |
234 | ||
235 | BlockDriverAIOCB *dma_bdrv_write(BlockDriverState *bs, | |
236 | QEMUSGList *sg, uint64_t sector, | |
237 | void (*cb)(void *opaque, int ret), void *opaque) | |
238 | { | |
43cf8ae6 DG |
239 | return dma_bdrv_io(bs, sg, sector, bdrv_aio_writev, cb, opaque, |
240 | DMA_DIRECTION_TO_DEVICE); | |
59a703eb | 241 | } |
8171ee35 PB |
242 | |
243 | ||
c65bcef3 DG |
244 | static uint64_t dma_buf_rw(uint8_t *ptr, int32_t len, QEMUSGList *sg, |
245 | DMADirection dir) | |
8171ee35 PB |
246 | { |
247 | uint64_t resid; | |
248 | int sg_cur_index; | |
249 | ||
250 | resid = sg->size; | |
251 | sg_cur_index = 0; | |
252 | len = MIN(len, resid); | |
253 | while (len > 0) { | |
254 | ScatterGatherEntry entry = sg->sg[sg_cur_index++]; | |
255 | int32_t xfer = MIN(len, entry.len); | |
c65bcef3 | 256 | dma_memory_rw(sg->dma, entry.base, ptr, xfer, dir); |
8171ee35 PB |
257 | ptr += xfer; |
258 | len -= xfer; | |
259 | resid -= xfer; | |
260 | } | |
261 | ||
262 | return resid; | |
263 | } | |
264 | ||
265 | uint64_t dma_buf_read(uint8_t *ptr, int32_t len, QEMUSGList *sg) | |
266 | { | |
c65bcef3 | 267 | return dma_buf_rw(ptr, len, sg, DMA_DIRECTION_FROM_DEVICE); |
8171ee35 PB |
268 | } |
269 | ||
270 | uint64_t dma_buf_write(uint8_t *ptr, int32_t len, QEMUSGList *sg) | |
271 | { | |
c65bcef3 | 272 | return dma_buf_rw(ptr, len, sg, DMA_DIRECTION_TO_DEVICE); |
8171ee35 | 273 | } |
84a69356 PB |
274 | |
275 | void dma_acct_start(BlockDriverState *bs, BlockAcctCookie *cookie, | |
276 | QEMUSGList *sg, enum BlockAcctType type) | |
277 | { | |
278 | bdrv_acct_start(bs, cookie, sg->size, type); | |
279 | } | |
e5332e63 DG |
280 | |
281 | bool iommu_dma_memory_valid(DMAContext *dma, dma_addr_t addr, dma_addr_t len, | |
282 | DMADirection dir) | |
283 | { | |
a8170e5e | 284 | hwaddr paddr, plen; |
e5332e63 DG |
285 | |
286 | #ifdef DEBUG_IOMMU | |
287 | fprintf(stderr, "dma_memory_check context=%p addr=0x" DMA_ADDR_FMT | |
288 | " len=0x" DMA_ADDR_FMT " dir=%d\n", dma, addr, len, dir); | |
289 | #endif | |
290 | ||
291 | while (len) { | |
292 | if (dma->translate(dma, addr, &paddr, &plen, dir) != 0) { | |
293 | return false; | |
294 | } | |
295 | ||
296 | /* The translation might be valid for larger regions. */ | |
297 | if (plen > len) { | |
298 | plen = len; | |
299 | } | |
300 | ||
301 | len -= plen; | |
302 | addr += plen; | |
303 | } | |
304 | ||
305 | return true; | |
306 | } | |
307 | ||
308 | int iommu_dma_memory_rw(DMAContext *dma, dma_addr_t addr, | |
309 | void *buf, dma_addr_t len, DMADirection dir) | |
310 | { | |
a8170e5e | 311 | hwaddr paddr, plen; |
e5332e63 DG |
312 | int err; |
313 | ||
314 | #ifdef DEBUG_IOMMU | |
315 | fprintf(stderr, "dma_memory_rw context=%p addr=0x" DMA_ADDR_FMT " len=0x" | |
316 | DMA_ADDR_FMT " dir=%d\n", dma, addr, len, dir); | |
317 | #endif | |
318 | ||
319 | while (len) { | |
320 | err = dma->translate(dma, addr, &paddr, &plen, dir); | |
321 | if (err) { | |
322 | /* | |
323 | * In case of failure on reads from the guest, we clean the | |
324 | * destination buffer so that a device that doesn't test | |
325 | * for errors will not expose qemu internal memory. | |
326 | */ | |
327 | memset(buf, 0, len); | |
328 | return -1; | |
329 | } | |
330 | ||
331 | /* The translation might be valid for larger regions. */ | |
332 | if (plen > len) { | |
333 | plen = len; | |
334 | } | |
335 | ||
b90600ee | 336 | address_space_rw(dma->as, paddr, buf, plen, dir == DMA_DIRECTION_FROM_DEVICE); |
e5332e63 DG |
337 | |
338 | len -= plen; | |
339 | addr += plen; | |
340 | buf += plen; | |
341 | } | |
342 | ||
343 | return 0; | |
344 | } | |
345 | ||
346 | int iommu_dma_memory_set(DMAContext *dma, dma_addr_t addr, uint8_t c, | |
347 | dma_addr_t len) | |
348 | { | |
a8170e5e | 349 | hwaddr paddr, plen; |
e5332e63 DG |
350 | int err; |
351 | ||
352 | #ifdef DEBUG_IOMMU | |
353 | fprintf(stderr, "dma_memory_set context=%p addr=0x" DMA_ADDR_FMT | |
354 | " len=0x" DMA_ADDR_FMT "\n", dma, addr, len); | |
355 | #endif | |
356 | ||
357 | while (len) { | |
358 | err = dma->translate(dma, addr, &paddr, &plen, | |
359 | DMA_DIRECTION_FROM_DEVICE); | |
360 | if (err) { | |
361 | return err; | |
362 | } | |
363 | ||
364 | /* The translation might be valid for larger regions. */ | |
365 | if (plen > len) { | |
366 | plen = len; | |
367 | } | |
368 | ||
b90600ee | 369 | do_dma_memory_set(dma->as, paddr, c, plen); |
e5332e63 DG |
370 | |
371 | len -= plen; | |
372 | addr += plen; | |
373 | } | |
374 | ||
375 | return 0; | |
376 | } | |
377 | ||
b90600ee | 378 | void dma_context_init(DMAContext *dma, AddressSpace *as, DMATranslateFunc translate, |
e5332e63 DG |
379 | DMAMapFunc map, DMAUnmapFunc unmap) |
380 | { | |
381 | #ifdef DEBUG_IOMMU | |
382 | fprintf(stderr, "dma_context_init(%p, %p, %p, %p)\n", | |
383 | dma, translate, map, unmap); | |
384 | #endif | |
b90600ee | 385 | dma->as = as; |
e5332e63 DG |
386 | dma->translate = translate; |
387 | dma->map = map; | |
388 | dma->unmap = unmap; | |
389 | } | |
390 | ||
391 | void *iommu_dma_memory_map(DMAContext *dma, dma_addr_t addr, dma_addr_t *len, | |
392 | DMADirection dir) | |
393 | { | |
394 | int err; | |
a8170e5e | 395 | hwaddr paddr, plen; |
e5332e63 DG |
396 | void *buf; |
397 | ||
398 | if (dma->map) { | |
399 | return dma->map(dma, addr, len, dir); | |
400 | } | |
401 | ||
402 | plen = *len; | |
403 | err = dma->translate(dma, addr, &paddr, &plen, dir); | |
404 | if (err) { | |
405 | return NULL; | |
406 | } | |
407 | ||
408 | /* | |
409 | * If this is true, the virtual region is contiguous, | |
410 | * but the translated physical region isn't. We just | |
b90600ee | 411 | * clamp *len, much like address_space_map() does. |
e5332e63 DG |
412 | */ |
413 | if (plen < *len) { | |
414 | *len = plen; | |
415 | } | |
416 | ||
b90600ee | 417 | buf = address_space_map(dma->as, paddr, &plen, dir == DMA_DIRECTION_FROM_DEVICE); |
e5332e63 DG |
418 | *len = plen; |
419 | ||
420 | return buf; | |
421 | } | |
422 | ||
423 | void iommu_dma_memory_unmap(DMAContext *dma, void *buffer, dma_addr_t len, | |
424 | DMADirection dir, dma_addr_t access_len) | |
425 | { | |
426 | if (dma->unmap) { | |
427 | dma->unmap(dma, buffer, len, dir, access_len); | |
428 | return; | |
429 | } | |
430 | ||
b90600ee AK |
431 | address_space_unmap(dma->as, buffer, len, dir == DMA_DIRECTION_FROM_DEVICE, |
432 | access_len); | |
e5332e63 DG |
433 | |
434 | } |