[mirror_qemu.git] / dma-helpers.c

/*
 * DMA helper functions
 *
 * Copyright (c) 2009 Red Hat
 *
 * This work is licensed under the terms of the GNU General Public License
 * (GNU GPL), version 2 or later.
 */

#include "dma.h"
#include "trace.h"

void qemu_sglist_init(QEMUSGList *qsg, int alloc_hint)
{
    qsg->sg = g_malloc(alloc_hint * sizeof(ScatterGatherEntry));
    qsg->nsg = 0;
    qsg->nalloc = alloc_hint;
    qsg->size = 0;
}

void qemu_sglist_add(QEMUSGList *qsg, dma_addr_t base, dma_addr_t len)
{
    if (qsg->nsg == qsg->nalloc) {
        qsg->nalloc = 2 * qsg->nalloc + 1;
        qsg->sg = g_realloc(qsg->sg, qsg->nalloc * sizeof(ScatterGatherEntry));
    }
    qsg->sg[qsg->nsg].base = base;
    qsg->sg[qsg->nsg].len = len;
    qsg->size += len;
    ++qsg->nsg;
}

void qemu_sglist_destroy(QEMUSGList *qsg)
{
    g_free(qsg->sg);
}

typedef struct {
    BlockDriverAIOCB common;
    BlockDriverState *bs;
    BlockDriverAIOCB *acb;
    QEMUSGList *sg;
    uint64_t sector_num;
    DMADirection dir;
    bool in_cancel;
    int sg_cur_index;
    dma_addr_t sg_cur_byte;
    QEMUIOVector iov;
    QEMUBH *bh;
    DMAIOFunc *io_func;
} DMAAIOCB;

static void dma_bdrv_cb(void *opaque, int ret);

static void reschedule_dma(void *opaque)
{
    DMAAIOCB *dbs = (DMAAIOCB *)opaque;

    qemu_bh_delete(dbs->bh);
    dbs->bh = NULL;
    dma_bdrv_cb(dbs, 0);
}

static void continue_after_map_failure(void *opaque)
{
    DMAAIOCB *dbs = (DMAAIOCB *)opaque;

    dbs->bh = qemu_bh_new(reschedule_dma, dbs);
    qemu_bh_schedule(dbs->bh);
}

static void dma_bdrv_unmap(DMAAIOCB *dbs)
{
    int i;

    for (i = 0; i < dbs->iov.niov; ++i) {
        cpu_physical_memory_unmap(dbs->iov.iov[i].iov_base,
                                  dbs->iov.iov[i].iov_len,
                                  dbs->dir != DMA_DIRECTION_TO_DEVICE,
                                  dbs->iov.iov[i].iov_len);
    }
    qemu_iovec_reset(&dbs->iov);
}

static void dma_complete(DMAAIOCB *dbs, int ret)
{
    trace_dma_complete(dbs, ret, dbs->common.cb);

    dma_bdrv_unmap(dbs);
    if (dbs->common.cb) {
        dbs->common.cb(dbs->common.opaque, ret);
    }
    qemu_iovec_destroy(&dbs->iov);
    if (dbs->bh) {
        qemu_bh_delete(dbs->bh);
        dbs->bh = NULL;
    }
    if (!dbs->in_cancel) {
        /* Requests may complete while dma_aio_cancel is in progress.  In
         * this case, the AIOCB should not be released because it is still
         * referenced by dma_aio_cancel.  */
        qemu_aio_release(dbs);
    }
}

static void dma_bdrv_cb(void *opaque, int ret)
{
    DMAAIOCB *dbs = (DMAAIOCB *)opaque;
    target_phys_addr_t cur_addr, cur_len;
    void *mem;

    trace_dma_bdrv_cb(dbs, ret);

    dbs->acb = NULL;
    dbs->sector_num += dbs->iov.size / 512;
    dma_bdrv_unmap(dbs);

    if (dbs->sg_cur_index == dbs->sg->nsg || ret < 0) {
        dma_complete(dbs, ret);
        return;
    }

    while (dbs->sg_cur_index < dbs->sg->nsg) {
        cur_addr = dbs->sg->sg[dbs->sg_cur_index].base + dbs->sg_cur_byte;
        cur_len = dbs->sg->sg[dbs->sg_cur_index].len - dbs->sg_cur_byte;
        mem = cpu_physical_memory_map(cur_addr, &cur_len,
                                      dbs->dir != DMA_DIRECTION_TO_DEVICE);
        if (!mem)
            break;
        qemu_iovec_add(&dbs->iov, mem, cur_len);
        dbs->sg_cur_byte += cur_len;
        if (dbs->sg_cur_byte == dbs->sg->sg[dbs->sg_cur_index].len) {
            dbs->sg_cur_byte = 0;
            ++dbs->sg_cur_index;
        }
    }

    if (dbs->iov.size == 0) {
        trace_dma_map_wait(dbs);
        cpu_register_map_client(dbs, continue_after_map_failure);
        return;
    }

    dbs->acb = dbs->io_func(dbs->bs, dbs->sector_num, &dbs->iov,
                            dbs->iov.size / 512, dma_bdrv_cb, dbs);
    assert(dbs->acb);
}

static void dma_aio_cancel(BlockDriverAIOCB *acb)
{
    DMAAIOCB *dbs = container_of(acb, DMAAIOCB, common);

    trace_dma_aio_cancel(dbs);

    if (dbs->acb) {
        BlockDriverAIOCB *acb = dbs->acb;
        dbs->acb = NULL;
        dbs->in_cancel = true;
        bdrv_aio_cancel(acb);
        dbs->in_cancel = false;
    }
    dbs->common.cb = NULL;
    dma_complete(dbs, 0);
}

static AIOPool dma_aio_pool = {
    .aiocb_size         = sizeof(DMAAIOCB),
    .cancel             = dma_aio_cancel,
};

BlockDriverAIOCB *dma_bdrv_io(
    BlockDriverState *bs, QEMUSGList *sg, uint64_t sector_num,
    DMAIOFunc *io_func, BlockDriverCompletionFunc *cb,
    void *opaque, DMADirection dir)
{
    DMAAIOCB *dbs = qemu_aio_get(&dma_aio_pool, bs, cb, opaque);

    trace_dma_bdrv_io(dbs, bs, sector_num, (dir == DMA_DIRECTION_TO_DEVICE));

    dbs->acb = NULL;
    dbs->bs = bs;
    dbs->sg = sg;
    dbs->sector_num = sector_num;
    dbs->sg_cur_index = 0;
    dbs->sg_cur_byte = 0;
    dbs->dir = dir;
    dbs->io_func = io_func;
    dbs->bh = NULL;
    qemu_iovec_init(&dbs->iov, sg->nsg);
    dma_bdrv_cb(dbs, 0);
    return &dbs->common;
}


BlockDriverAIOCB *dma_bdrv_read(BlockDriverState *bs,
                                QEMUSGList *sg, uint64_t sector,
                                void (*cb)(void *opaque, int ret), void *opaque)
{
    return dma_bdrv_io(bs, sg, sector, bdrv_aio_readv, cb, opaque,
                       DMA_DIRECTION_FROM_DEVICE);
}

BlockDriverAIOCB *dma_bdrv_write(BlockDriverState *bs,
                                 QEMUSGList *sg, uint64_t sector,
                                 void (*cb)(void *opaque, int ret), void *opaque)
{
    return dma_bdrv_io(bs, sg, sector, bdrv_aio_writev, cb, opaque,
                       DMA_DIRECTION_TO_DEVICE);
}


static uint64_t dma_buf_rw(uint8_t *ptr, int32_t len, QEMUSGList *sg, bool to_dev)
{
    uint64_t resid;
    int sg_cur_index;

    resid = sg->size;
    sg_cur_index = 0;
    len = MIN(len, resid);
    while (len > 0) {
        ScatterGatherEntry entry = sg->sg[sg_cur_index++];
        int32_t xfer = MIN(len, entry.len);
        cpu_physical_memory_rw(entry.base, ptr, xfer, !to_dev);
        ptr += xfer;
        len -= xfer;
        resid -= xfer;
    }

    return resid;
}

uint64_t dma_buf_read(uint8_t *ptr, int32_t len, QEMUSGList *sg)
{
    return dma_buf_rw(ptr, len, sg, 0);
}

uint64_t dma_buf_write(uint8_t *ptr, int32_t len, QEMUSGList *sg)
{
    return dma_buf_rw(ptr, len, sg, 1);
}

void dma_acct_start(BlockDriverState *bs, BlockAcctCookie *cookie,
                    QEMUSGList *sg, enum BlockAcctType type)
{
    bdrv_acct_start(bs, cookie, sg->size, type);
}
Commit	Line	Data
	1	/*
	2	* DMA helper functions
	3	*
	4	* Copyright (c) 2009 Red Hat
	5	*
	6	* This work is licensed under the terms of the GNU General Public License
	7	* (GNU GPL), version 2 or later.
	8	*/
	9
	10	#include "dma.h"
	11	#include "trace.h"
	12
	13	void qemu_sglist_init(QEMUSGList *qsg, int alloc_hint)
	14	{
	15	qsg->sg = g_malloc(alloc_hint * sizeof(ScatterGatherEntry));
	16	qsg->nsg = 0;
	17	qsg->nalloc = alloc_hint;
	18	qsg->size = 0;
	19	}
	20
	21	void qemu_sglist_add(QEMUSGList *qsg, dma_addr_t base, dma_addr_t len)
	22	{
	23	if (qsg->nsg == qsg->nalloc) {
	24	qsg->nalloc = 2 * qsg->nalloc + 1;
	25	qsg->sg = g_realloc(qsg->sg, qsg->nalloc * sizeof(ScatterGatherEntry));
	26	}
	27	qsg->sg[qsg->nsg].base = base;
	28	qsg->sg[qsg->nsg].len = len;
	29	qsg->size += len;
	30	++qsg->nsg;
	31	}
	32
	33	void qemu_sglist_destroy(QEMUSGList *qsg)
	34	{
	35	g_free(qsg->sg);
	36	}
	37
	38	typedef struct {
	39	BlockDriverAIOCB common;
	40	BlockDriverState *bs;
	41	BlockDriverAIOCB *acb;
	42	QEMUSGList *sg;
	43	uint64_t sector_num;
	44	DMADirection dir;
	45	bool in_cancel;
	46	int sg_cur_index;
	47	dma_addr_t sg_cur_byte;
	48	QEMUIOVector iov;
	49	QEMUBH *bh;
	50	DMAIOFunc *io_func;
	51	} DMAAIOCB;
	52
	53	static void dma_bdrv_cb(void *opaque, int ret);
	54
	55	static void reschedule_dma(void *opaque)
	56	{
	57	DMAAIOCB dbs = (DMAAIOCB )opaque;
	58
	59	qemu_bh_delete(dbs->bh);
	60	dbs->bh = NULL;
	61	dma_bdrv_cb(dbs, 0);
	62	}
	63
	64	static void continue_after_map_failure(void *opaque)
	65	{
	66	DMAAIOCB dbs = (DMAAIOCB )opaque;
	67
	68	dbs->bh = qemu_bh_new(reschedule_dma, dbs);
	69	qemu_bh_schedule(dbs->bh);
	70	}
	71
	72	static void dma_bdrv_unmap(DMAAIOCB *dbs)
	73	{
	74	int i;
	75
	76	for (i = 0; i < dbs->iov.niov; ++i) {
	77	cpu_physical_memory_unmap(dbs->iov.iov[i].iov_base,
	78	dbs->iov.iov[i].iov_len,
	79	dbs->dir != DMA_DIRECTION_TO_DEVICE,
	80	dbs->iov.iov[i].iov_len);
	81	}
	82	qemu_iovec_reset(&dbs->iov);
	83	}
	84
	85	static void dma_complete(DMAAIOCB *dbs, int ret)
	86	{
	87	trace_dma_complete(dbs, ret, dbs->common.cb);
	88
	89	dma_bdrv_unmap(dbs);
	90	if (dbs->common.cb) {
	91	dbs->common.cb(dbs->common.opaque, ret);
	92	}
	93	qemu_iovec_destroy(&dbs->iov);
	94	if (dbs->bh) {
	95	qemu_bh_delete(dbs->bh);
	96	dbs->bh = NULL;
	97	}
	98	if (!dbs->in_cancel) {
	99	/* Requests may complete while dma_aio_cancel is in progress. In
	100	* this case, the AIOCB should not be released because it is still
	101	* referenced by dma_aio_cancel. */
	102	qemu_aio_release(dbs);
	103	}
	104	}
	105
	106	static void dma_bdrv_cb(void *opaque, int ret)
	107	{
	108	DMAAIOCB dbs = (DMAAIOCB )opaque;
	109	target_phys_addr_t cur_addr, cur_len;
	110	void *mem;
	111
	112	trace_dma_bdrv_cb(dbs, ret);
	113
	114	dbs->acb = NULL;
	115	dbs->sector_num += dbs->iov.size / 512;
	116	dma_bdrv_unmap(dbs);
	117
	118	if (dbs->sg_cur_index == dbs->sg->nsg \|\| ret < 0) {
	119	dma_complete(dbs, ret);
	120	return;
	121	}
	122
	123	while (dbs->sg_cur_index < dbs->sg->nsg) {
	124	cur_addr = dbs->sg->sg[dbs->sg_cur_index].base + dbs->sg_cur_byte;
	125	cur_len = dbs->sg->sg[dbs->sg_cur_index].len - dbs->sg_cur_byte;
	126	mem = cpu_physical_memory_map(cur_addr, &cur_len,
	127	dbs->dir != DMA_DIRECTION_TO_DEVICE);
	128	if (!mem)
	129	break;
	130	qemu_iovec_add(&dbs->iov, mem, cur_len);
	131	dbs->sg_cur_byte += cur_len;
	132	if (dbs->sg_cur_byte == dbs->sg->sg[dbs->sg_cur_index].len) {
	133	dbs->sg_cur_byte = 0;
	134	++dbs->sg_cur_index;
	135	}
	136	}
	137
	138	if (dbs->iov.size == 0) {
	139	trace_dma_map_wait(dbs);
	140	cpu_register_map_client(dbs, continue_after_map_failure);
	141	return;
	142	}
	143
	144	dbs->acb = dbs->io_func(dbs->bs, dbs->sector_num, &dbs->iov,
	145	dbs->iov.size / 512, dma_bdrv_cb, dbs);
	146	assert(dbs->acb);
	147	}
	148
	149	static void dma_aio_cancel(BlockDriverAIOCB *acb)
	150	{
	151	DMAAIOCB *dbs = container_of(acb, DMAAIOCB, common);
	152
	153	trace_dma_aio_cancel(dbs);
	154
	155	if (dbs->acb) {
	156	BlockDriverAIOCB *acb = dbs->acb;
	157	dbs->acb = NULL;
	158	dbs->in_cancel = true;
	159	bdrv_aio_cancel(acb);
	160	dbs->in_cancel = false;
	161	}
	162	dbs->common.cb = NULL;
	163	dma_complete(dbs, 0);
	164	}
	165
	166	static AIOPool dma_aio_pool = {
	167	.aiocb_size = sizeof(DMAAIOCB),
	168	.cancel = dma_aio_cancel,
	169	};
	170
	171	BlockDriverAIOCB *dma_bdrv_io(
	172	BlockDriverState bs, QEMUSGList sg, uint64_t sector_num,
	173	DMAIOFunc io_func, BlockDriverCompletionFunc cb,
	174	void *opaque, DMADirection dir)
	175	{
	176	DMAAIOCB *dbs = qemu_aio_get(&dma_aio_pool, bs, cb, opaque);
	177
	178	trace_dma_bdrv_io(dbs, bs, sector_num, (dir == DMA_DIRECTION_TO_DEVICE));
	179
	180	dbs->acb = NULL;
	181	dbs->bs = bs;
	182	dbs->sg = sg;
	183	dbs->sector_num = sector_num;
	184	dbs->sg_cur_index = 0;
	185	dbs->sg_cur_byte = 0;
	186	dbs->dir = dir;
	187	dbs->io_func = io_func;
	188	dbs->bh = NULL;
	189	qemu_iovec_init(&dbs->iov, sg->nsg);
	190	dma_bdrv_cb(dbs, 0);
	191	return &dbs->common;
	192	}
	193
	194
	195	BlockDriverAIOCB dma_bdrv_read(BlockDriverState bs,
	196	QEMUSGList *sg, uint64_t sector,
	197	void (cb)(void opaque, int ret), void *opaque)
	198	{
	199	return dma_bdrv_io(bs, sg, sector, bdrv_aio_readv, cb, opaque,
	200	DMA_DIRECTION_FROM_DEVICE);
	201	}
	202
	203	BlockDriverAIOCB dma_bdrv_write(BlockDriverState bs,
	204	QEMUSGList *sg, uint64_t sector,
	205	void (cb)(void opaque, int ret), void *opaque)
	206	{
	207	return dma_bdrv_io(bs, sg, sector, bdrv_aio_writev, cb, opaque,
	208	DMA_DIRECTION_TO_DEVICE);
	209	}
	210
	211
	212	static uint64_t dma_buf_rw(uint8_t ptr, int32_t len, QEMUSGList sg, bool to_dev)
	213	{
	214	uint64_t resid;
	215	int sg_cur_index;
	216
	217	resid = sg->size;
	218	sg_cur_index = 0;
	219	len = MIN(len, resid);
	220	while (len > 0) {
	221	ScatterGatherEntry entry = sg->sg[sg_cur_index++];
	222	int32_t xfer = MIN(len, entry.len);
	223	cpu_physical_memory_rw(entry.base, ptr, xfer, !to_dev);
	224	ptr += xfer;
	225	len -= xfer;
	226	resid -= xfer;
	227	}
	228
	229	return resid;
	230	}
	231
	232	uint64_t dma_buf_read(uint8_t ptr, int32_t len, QEMUSGList sg)
	233	{
	234	return dma_buf_rw(ptr, len, sg, 0);
	235	}
	236
	237	uint64_t dma_buf_write(uint8_t ptr, int32_t len, QEMUSGList sg)
	238	{
	239	return dma_buf_rw(ptr, len, sg, 1);
	240	}
	241
	242	void dma_acct_start(BlockDriverState bs, BlockAcctCookie cookie,
	243	QEMUSGList *sg, enum BlockAcctType type)
	244	{
	245	bdrv_acct_start(bs, cookie, sg->size, type);
	246	}