dma-helpers.c

   1 /*
   2  * DMA helper functions
   3  *
   4  * Copyright (c) 2009 Red Hat
   5  *
   6  * This work is licensed under the terms of the GNU General Public License
   7  * (GNU GPL), version 2 or later.
   8  */
   9
  10 #include "sysemu/dma.h"
  11 #include "trace.h"
  12 #include "qemu/range.h"
  13 #include "qemu/thread.h"
  14
  15 /* #define DEBUG_IOMMU */
  16
  17 int dma_memory_set(DMAContext *dma, dma_addr_t addr, uint8_t c, dma_addr_t len)
  18 {
  19     AddressSpace *as = dma->as;
  20
  21     dma_barrier(dma, DMA_DIRECTION_FROM_DEVICE);
  22
  23 #define FILLBUF_SIZE 512
  24     uint8_t fillbuf[FILLBUF_SIZE];
  25     int l;
  26     bool error = false;
  27
  28     memset(fillbuf, c, FILLBUF_SIZE);
  29     while (len > 0) {
  30         l = len < FILLBUF_SIZE ? len : FILLBUF_SIZE;
  31         error |= address_space_rw(as, addr, fillbuf, l, true);
  32         len -= l;
  33         addr += l;
  34     }
  35
  36     return error;
  37 }
  38
  39 void qemu_sglist_init(QEMUSGList *qsg, int alloc_hint, DMAContext *dma)
  40 {
  41     qsg->sg = g_malloc(alloc_hint * sizeof(ScatterGatherEntry));
  42     qsg->nsg = 0;
  43     qsg->nalloc = alloc_hint;
  44     qsg->size = 0;
  45     qsg->dma = dma;
  46 }
  47
  48 void qemu_sglist_add(QEMUSGList *qsg, dma_addr_t base, dma_addr_t len)
  49 {
  50     if (qsg->nsg == qsg->nalloc) {
  51         qsg->nalloc = 2 * qsg->nalloc + 1;
  52         qsg->sg = g_realloc(qsg->sg, qsg->nalloc * sizeof(ScatterGatherEntry));
  53     }
  54     qsg->sg[qsg->nsg].base = base;
  55     qsg->sg[qsg->nsg].len = len;
  56     qsg->size += len;
  57     ++qsg->nsg;
  58 }
  59
  60 void qemu_sglist_destroy(QEMUSGList *qsg)
  61 {
  62     g_free(qsg->sg);
  63     memset(qsg, 0, sizeof(*qsg));
  64 }
  65
  66 typedef struct {
  67     BlockDriverAIOCB common;
  68     BlockDriverState *bs;
  69     BlockDriverAIOCB *acb;
  70     QEMUSGList *sg;
  71     uint64_t sector_num;
  72     DMADirection dir;
  73     bool in_cancel;
  74     int sg_cur_index;
  75     dma_addr_t sg_cur_byte;
  76     QEMUIOVector iov;
  77     QEMUBH *bh;
  78     DMAIOFunc *io_func;
  79 } DMAAIOCB;
  80
  81 static void dma_bdrv_cb(void *opaque, int ret);
  82
  83 static void reschedule_dma(void *opaque)
  84 {
  85     DMAAIOCB *dbs = (DMAAIOCB *)opaque;
  86
  87     qemu_bh_delete(dbs->bh);
  88     dbs->bh = NULL;
  89     dma_bdrv_cb(dbs, 0);
  90 }
  91
  92 static void continue_after_map_failure(void *opaque)
  93 {
  94     DMAAIOCB *dbs = (DMAAIOCB *)opaque;
  95
  96     dbs->bh = qemu_bh_new(reschedule_dma, dbs);
  97     qemu_bh_schedule(dbs->bh);
  98 }
  99
 100 static void dma_bdrv_unmap(DMAAIOCB *dbs)
 101 {
 102     int i;
 103
 104     for (i = 0; i < dbs->iov.niov; ++i) {
 105         dma_memory_unmap(dbs->sg->dma, dbs->iov.iov[i].iov_base,
 106                          dbs->iov.iov[i].iov_len, dbs->dir,
 107                          dbs->iov.iov[i].iov_len);
 108     }
 109     qemu_iovec_reset(&dbs->iov);
 110 }
 111
 112 static void dma_complete(DMAAIOCB *dbs, int ret)
 113 {
 114     trace_dma_complete(dbs, ret, dbs->common.cb);
 115
 116     dma_bdrv_unmap(dbs);
 117     if (dbs->common.cb) {
 118         dbs->common.cb(dbs->common.opaque, ret);
 119     }
 120     qemu_iovec_destroy(&dbs->iov);
 121     if (dbs->bh) {
 122         qemu_bh_delete(dbs->bh);
 123         dbs->bh = NULL;
 124     }
 125     if (!dbs->in_cancel) {
 126         /* Requests may complete while dma_aio_cancel is in progress.  In
 127          * this case, the AIOCB should not be released because it is still
 128          * referenced by dma_aio_cancel.  */
 129         qemu_aio_release(dbs);
 130     }
 131 }
 132
 133 static void dma_bdrv_cb(void *opaque, int ret)
 134 {
 135     DMAAIOCB *dbs = (DMAAIOCB *)opaque;
 136     dma_addr_t cur_addr, cur_len;
 137     void *mem;
 138
 139     trace_dma_bdrv_cb(dbs, ret);
 140
 141     dbs->acb = NULL;
 142     dbs->sector_num += dbs->iov.size / 512;
 143     dma_bdrv_unmap(dbs);
 144
 145     if (dbs->sg_cur_index == dbs->sg->nsg || ret < 0) {
 146         dma_complete(dbs, ret);
 147         return;
 148     }
 149
 150     while (dbs->sg_cur_index < dbs->sg->nsg) {
 151         cur_addr = dbs->sg->sg[dbs->sg_cur_index].base + dbs->sg_cur_byte;
 152         cur_len = dbs->sg->sg[dbs->sg_cur_index].len - dbs->sg_cur_byte;
 153         mem = dma_memory_map(dbs->sg->dma, cur_addr, &cur_len, dbs->dir);
 154         if (!mem)
 155             break;
 156         qemu_iovec_add(&dbs->iov, mem, cur_len);
 157         dbs->sg_cur_byte += cur_len;
 158         if (dbs->sg_cur_byte == dbs->sg->sg[dbs->sg_cur_index].len) {
 159             dbs->sg_cur_byte = 0;
 160             ++dbs->sg_cur_index;
 161         }
 162     }
 163
 164     if (dbs->iov.size == 0) {
 165         trace_dma_map_wait(dbs);
 166         cpu_register_map_client(dbs, continue_after_map_failure);
 167         return;
 168     }
 169
 170     dbs->acb = dbs->io_func(dbs->bs, dbs->sector_num, &dbs->iov,
 171                             dbs->iov.size / 512, dma_bdrv_cb, dbs);
 172     assert(dbs->acb);
 173 }
 174
 175 static void dma_aio_cancel(BlockDriverAIOCB *acb)
 176 {
 177     DMAAIOCB *dbs = container_of(acb, DMAAIOCB, common);
 178
 179     trace_dma_aio_cancel(dbs);
 180
 181     if (dbs->acb) {
 182         BlockDriverAIOCB *acb = dbs->acb;
 183         dbs->acb = NULL;
 184         dbs->in_cancel = true;
 185         bdrv_aio_cancel(acb);
 186         dbs->in_cancel = false;
 187     }
 188     dbs->common.cb = NULL;
 189     dma_complete(dbs, 0);
 190 }
 191
 192 static const AIOCBInfo dma_aiocb_info = {
 193     .aiocb_size         = sizeof(DMAAIOCB),
 194     .cancel             = dma_aio_cancel,
 195 };
 196
 197 BlockDriverAIOCB *dma_bdrv_io(
 198     BlockDriverState *bs, QEMUSGList *sg, uint64_t sector_num,
 199     DMAIOFunc *io_func, BlockDriverCompletionFunc *cb,
 200     void *opaque, DMADirection dir)
 201 {
 202     DMAAIOCB *dbs = qemu_aio_get(&dma_aiocb_info, bs, cb, opaque);
 203
 204     trace_dma_bdrv_io(dbs, bs, sector_num, (dir == DMA_DIRECTION_TO_DEVICE));
 205
 206     dbs->acb = NULL;
 207     dbs->bs = bs;
 208     dbs->sg = sg;
 209     dbs->sector_num = sector_num;
 210     dbs->sg_cur_index = 0;
 211     dbs->sg_cur_byte = 0;
 212     dbs->dir = dir;
 213     dbs->io_func = io_func;
 214     dbs->bh = NULL;
 215     qemu_iovec_init(&dbs->iov, sg->nsg);
 216     dma_bdrv_cb(dbs, 0);
 217     return &dbs->common;
 218 }
 219
 220
 221 BlockDriverAIOCB *dma_bdrv_read(BlockDriverState *bs,
 222                                 QEMUSGList *sg, uint64_t sector,
 223                                 void (*cb)(void *opaque, int ret), void *opaque)
 224 {
 225     return dma_bdrv_io(bs, sg, sector, bdrv_aio_readv, cb, opaque,
 226                        DMA_DIRECTION_FROM_DEVICE);
 227 }
 228
 229 BlockDriverAIOCB *dma_bdrv_write(BlockDriverState *bs,
 230                                  QEMUSGList *sg, uint64_t sector,
 231                                  void (*cb)(void *opaque, int ret), void *opaque)
 232 {
 233     return dma_bdrv_io(bs, sg, sector, bdrv_aio_writev, cb, opaque,
 234                        DMA_DIRECTION_TO_DEVICE);
 235 }
 236
 237
 238 static uint64_t dma_buf_rw(uint8_t *ptr, int32_t len, QEMUSGList *sg,
 239                            DMADirection dir)
 240 {
 241     uint64_t resid;
 242     int sg_cur_index;
 243
 244     resid = sg->size;
 245     sg_cur_index = 0;
 246     len = MIN(len, resid);
 247     while (len > 0) {
 248         ScatterGatherEntry entry = sg->sg[sg_cur_index++];
 249         int32_t xfer = MIN(len, entry.len);
 250         dma_memory_rw(sg->dma, entry.base, ptr, xfer, dir);
 251         ptr += xfer;
 252         len -= xfer;
 253         resid -= xfer;
 254     }
 255
 256     return resid;
 257 }
 258
 259 uint64_t dma_buf_read(uint8_t *ptr, int32_t len, QEMUSGList *sg)
 260 {
 261     return dma_buf_rw(ptr, len, sg, DMA_DIRECTION_FROM_DEVICE);
 262 }
 263
 264 uint64_t dma_buf_write(uint8_t *ptr, int32_t len, QEMUSGList *sg)
 265 {
 266     return dma_buf_rw(ptr, len, sg, DMA_DIRECTION_TO_DEVICE);
 267 }
 268
 269 void dma_acct_start(BlockDriverState *bs, BlockAcctCookie *cookie,
 270                     QEMUSGList *sg, enum BlockAcctType type)
 271 {
 272     bdrv_acct_start(bs, cookie, sg->size, type);
 273 }
 274
 275 void dma_context_init(DMAContext *dma, AddressSpace *as)
 276 {
 277 #ifdef DEBUG_IOMMU
 278     fprintf(stderr, "dma_context_init(%p -> %p)\n", dma, as);
 279 #endif
 280     dma->as = as;
 281 }