[qemu.git] / xen-mapcache.c

/*
 * Copyright (C) 2011       Citrix Ltd.
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 */

#include "config.h"

#include <sys/resource.h>

#include "hw/xen_backend.h"
#include "blockdev.h"
#include "bitmap.h"

#include <xen/hvm/params.h>
#include <sys/mman.h>

#include "xen-mapcache.h"
#include "trace.h"


//#define MAPCACHE_DEBUG

#ifdef MAPCACHE_DEBUG
#  define DPRINTF(fmt, ...) do { \
    fprintf(stderr, "xen_mapcache: " fmt, ## __VA_ARGS__); \
} while (0)
#else
#  define DPRINTF(fmt, ...) do { } while (0)
#endif

#if defined(__i386__)
#  define MCACHE_BUCKET_SHIFT 16
#  define MCACHE_MAX_SIZE     (1UL<<31) /* 2GB Cap */
#elif defined(__x86_64__)
#  define MCACHE_BUCKET_SHIFT 20
#  define MCACHE_MAX_SIZE     (1UL<<35) /* 32GB Cap */
#endif
#define MCACHE_BUCKET_SIZE (1UL << MCACHE_BUCKET_SHIFT)

#define mapcache_lock()   ((void)0)
#define mapcache_unlock() ((void)0)

typedef struct MapCacheEntry {
    target_phys_addr_t paddr_index;
    uint8_t *vaddr_base;
    unsigned long *valid_mapping;
    uint8_t lock;
    target_phys_addr_t size;
    struct MapCacheEntry *next;
} MapCacheEntry;

typedef struct MapCacheRev {
    uint8_t *vaddr_req;
    target_phys_addr_t paddr_index;
    target_phys_addr_t size;
    QTAILQ_ENTRY(MapCacheRev) next;
} MapCacheRev;

typedef struct MapCache {
    MapCacheEntry *entry;
    unsigned long nr_buckets;
    QTAILQ_HEAD(map_cache_head, MapCacheRev) locked_entries;

    /* For most cases (>99.9%), the page address is the same. */
    target_phys_addr_t last_address_index;
    uint8_t *last_address_vaddr;
    unsigned long max_mcache_size;
    unsigned int mcache_bucket_shift;
} MapCache;

static MapCache *mapcache;

static inline int test_bits(int nr, int size, const unsigned long *addr)
{
    unsigned long res = find_next_zero_bit(addr, size + nr, nr);
    if (res >= nr + size)
        return 1;
    else
        return 0;
}

void xen_map_cache_init(void)
{
    unsigned long size;
    struct rlimit rlimit_as;

    mapcache = qemu_mallocz(sizeof (MapCache));

    QTAILQ_INIT(&mapcache->locked_entries);
    mapcache->last_address_index = -1;

    getrlimit(RLIMIT_AS, &rlimit_as);
    if (rlimit_as.rlim_max < MCACHE_MAX_SIZE) {
        rlimit_as.rlim_cur = rlimit_as.rlim_max;
    } else {
        rlimit_as.rlim_cur = MCACHE_MAX_SIZE;
    }

    setrlimit(RLIMIT_AS, &rlimit_as);
    mapcache->max_mcache_size = rlimit_as.rlim_cur;

    mapcache->nr_buckets =
        (((mapcache->max_mcache_size >> XC_PAGE_SHIFT) +
          (1UL << (MCACHE_BUCKET_SHIFT - XC_PAGE_SHIFT)) - 1) >>
         (MCACHE_BUCKET_SHIFT - XC_PAGE_SHIFT));

    size = mapcache->nr_buckets * sizeof (MapCacheEntry);
    size = (size + XC_PAGE_SIZE - 1) & ~(XC_PAGE_SIZE - 1);
    DPRINTF("%s, nr_buckets = %lx size %lu\n", __func__,
            mapcache->nr_buckets, size);
    mapcache->entry = qemu_mallocz(size);
}

static void xen_remap_bucket(MapCacheEntry *entry,
                             target_phys_addr_t size,
                             target_phys_addr_t address_index)
{
    uint8_t *vaddr_base;
    xen_pfn_t *pfns;
    int *err;
    unsigned int i;
    target_phys_addr_t nb_pfn = size >> XC_PAGE_SHIFT;

    trace_xen_remap_bucket(address_index);

    pfns = qemu_mallocz(nb_pfn * sizeof (xen_pfn_t));
    err = qemu_mallocz(nb_pfn * sizeof (int));

    if (entry->vaddr_base != NULL) {
        if (munmap(entry->vaddr_base, entry->size) != 0) {
            perror("unmap fails");
            exit(-1);
        }
    }
    if (entry->valid_mapping != NULL) {
        qemu_free(entry->valid_mapping);
        entry->valid_mapping = NULL;
    }

    for (i = 0; i < nb_pfn; i++) {
        pfns[i] = (address_index << (MCACHE_BUCKET_SHIFT-XC_PAGE_SHIFT)) + i;
    }

    vaddr_base = xc_map_foreign_bulk(xen_xc, xen_domid, PROT_READ|PROT_WRITE,
                                     pfns, err, nb_pfn);
    if (vaddr_base == NULL) {
        perror("xc_map_foreign_bulk");
        exit(-1);
    }

    entry->vaddr_base = vaddr_base;
    entry->paddr_index = address_index;
    entry->size = size;
    entry->valid_mapping = (unsigned long *) qemu_mallocz(sizeof(unsigned long) *
            BITS_TO_LONGS(size >> XC_PAGE_SHIFT));

    bitmap_zero(entry->valid_mapping, nb_pfn);
    for (i = 0; i < nb_pfn; i++) {
        if (!err[i]) {
            bitmap_set(entry->valid_mapping, i, 1);
        }
    }

    qemu_free(pfns);
    qemu_free(err);
}

uint8_t *xen_map_cache(target_phys_addr_t phys_addr, target_phys_addr_t size,
                       uint8_t lock)
{
    MapCacheEntry *entry, *pentry = NULL;
    target_phys_addr_t address_index  = phys_addr >> MCACHE_BUCKET_SHIFT;
    target_phys_addr_t address_offset = phys_addr & (MCACHE_BUCKET_SIZE - 1);
    target_phys_addr_t __size = size;

    trace_xen_map_cache(phys_addr);

    if (address_index == mapcache->last_address_index && !lock && !__size) {
        trace_xen_map_cache_return(mapcache->last_address_vaddr + address_offset);
        return mapcache->last_address_vaddr + address_offset;
    }

    /* size is always a multiple of MCACHE_BUCKET_SIZE */
    if ((address_offset + (__size % MCACHE_BUCKET_SIZE)) > MCACHE_BUCKET_SIZE)
        __size += MCACHE_BUCKET_SIZE;
    if (__size % MCACHE_BUCKET_SIZE)
        __size += MCACHE_BUCKET_SIZE - (__size % MCACHE_BUCKET_SIZE);
    if (!__size)
        __size = MCACHE_BUCKET_SIZE;

    entry = &mapcache->entry[address_index % mapcache->nr_buckets];

    while (entry && entry->lock && entry->vaddr_base &&
            (entry->paddr_index != address_index || entry->size != __size ||
             !test_bits(address_offset >> XC_PAGE_SHIFT, size >> XC_PAGE_SHIFT,
                 entry->valid_mapping))) {
        pentry = entry;
        entry = entry->next;
    }
    if (!entry) {
        entry = qemu_mallocz(sizeof (MapCacheEntry));
        pentry->next = entry;
        xen_remap_bucket(entry, __size, address_index);
    } else if (!entry->lock) {
        if (!entry->vaddr_base || entry->paddr_index != address_index ||
                entry->size != __size ||
                !test_bits(address_offset >> XC_PAGE_SHIFT, size >> XC_PAGE_SHIFT,
                    entry->valid_mapping)) {
            xen_remap_bucket(entry, __size, address_index);
        }
    }

    if(!test_bits(address_offset >> XC_PAGE_SHIFT, size >> XC_PAGE_SHIFT,
                entry->valid_mapping)) {
        mapcache->last_address_index = -1;
        trace_xen_map_cache_return(NULL);
        return NULL;
    }

    mapcache->last_address_index = address_index;
    mapcache->last_address_vaddr = entry->vaddr_base;
    if (lock) {
        MapCacheRev *reventry = qemu_mallocz(sizeof(MapCacheRev));
        entry->lock++;
        reventry->vaddr_req = mapcache->last_address_vaddr + address_offset;
        reventry->paddr_index = mapcache->last_address_index;
        reventry->size = entry->size;
        QTAILQ_INSERT_HEAD(&mapcache->locked_entries, reventry, next);
    }

    trace_xen_map_cache_return(mapcache->last_address_vaddr + address_offset);
    return mapcache->last_address_vaddr + address_offset;
}

ram_addr_t xen_ram_addr_from_mapcache(void *ptr)
{
    MapCacheEntry *entry = NULL, *pentry = NULL;
    MapCacheRev *reventry;
    target_phys_addr_t paddr_index;
    target_phys_addr_t size;
    int found = 0;

    QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
        if (reventry->vaddr_req == ptr) {
            paddr_index = reventry->paddr_index;
            size = reventry->size;
            found = 1;
            break;
        }
    }
    if (!found) {
        fprintf(stderr, "%s, could not find %p\n", __func__, ptr);
        QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
            DPRINTF("   "TARGET_FMT_plx" -> %p is present\n", reventry->paddr_index,
                    reventry->vaddr_req);
        }
        abort();
        return 0;
    }

    entry = &mapcache->entry[paddr_index % mapcache->nr_buckets];
    while (entry && (entry->paddr_index != paddr_index || entry->size != size)) {
        pentry = entry;
        entry = entry->next;
    }
    if (!entry) {
        DPRINTF("Trying to find address %p that is not in the mapcache!\n", ptr);
        return 0;
    }
    return (reventry->paddr_index << MCACHE_BUCKET_SHIFT) +
        ((unsigned long) ptr - (unsigned long) entry->vaddr_base);
}

void xen_invalidate_map_cache_entry(uint8_t *buffer)
{
    MapCacheEntry *entry = NULL, *pentry = NULL;
    MapCacheRev *reventry;
    target_phys_addr_t paddr_index;
    target_phys_addr_t size;
    int found = 0;

    if (mapcache->last_address_vaddr == buffer) {
        mapcache->last_address_index = -1;
    }

    QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
        if (reventry->vaddr_req == buffer) {
            paddr_index = reventry->paddr_index;
            size = reventry->size;
            found = 1;
            break;
        }
    }
    if (!found) {
        DPRINTF("%s, could not find %p\n", __func__, buffer);
        QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
            DPRINTF("   "TARGET_FMT_plx" -> %p is present\n", reventry->paddr_index, reventry->vaddr_req);
        }
        return;
    }
    QTAILQ_REMOVE(&mapcache->locked_entries, reventry, next);
    qemu_free(reventry);

    entry = &mapcache->entry[paddr_index % mapcache->nr_buckets];
    while (entry && (entry->paddr_index != paddr_index || entry->size != size)) {
        pentry = entry;
        entry = entry->next;
    }
    if (!entry) {
        DPRINTF("Trying to unmap address %p that is not in the mapcache!\n", buffer);
        return;
    }
    entry->lock--;
    if (entry->lock > 0 || pentry == NULL) {
        return;
    }

    pentry->next = entry->next;
    if (munmap(entry->vaddr_base, entry->size) != 0) {
        perror("unmap fails");
        exit(-1);
    }
    qemu_free(entry->valid_mapping);
    qemu_free(entry);
}

void xen_invalidate_map_cache(void)
{
    unsigned long i;
    MapCacheRev *reventry;

    /* Flush pending AIO before destroying the mapcache */
    qemu_aio_flush();

    QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
        DPRINTF("There should be no locked mappings at this time, "
                "but "TARGET_FMT_plx" -> %p is present\n",
                reventry->paddr_index, reventry->vaddr_req);
    }

    mapcache_lock();

    for (i = 0; i < mapcache->nr_buckets; i++) {
        MapCacheEntry *entry = &mapcache->entry[i];

        if (entry->vaddr_base == NULL) {
            continue;
        }

        if (munmap(entry->vaddr_base, entry->size) != 0) {
            perror("unmap fails");
            exit(-1);
        }

        entry->paddr_index = 0;
        entry->vaddr_base = NULL;
        entry->size = 0;
        qemu_free(entry->valid_mapping);
        entry->valid_mapping = NULL;
    }

    mapcache->last_address_index = -1;
    mapcache->last_address_vaddr = NULL;

    mapcache_unlock();
}
Commit	Line	Data
432d268c JN	1	/*
	2	* Copyright (C) 2011 Citrix Ltd.
	3	*
	4	* This work is licensed under the terms of the GNU GPL, version 2. See
	5	* the COPYING file in the top-level directory.
	6	*
	7	*/
	8
	9	#include "config.h"
	10
	11	#include <sys/resource.h>
	12
	13	#include "hw/xen_backend.h"
	14	#include "blockdev.h"
ea6c5f8f	15	#include "bitmap.h"
432d268c JN	16
	17	#include <xen/hvm/params.h>
	18	#include <sys/mman.h>
	19
	20	#include "xen-mapcache.h"
	21	#include "trace.h"
	22
	23
	24	//#define MAPCACHE_DEBUG
	25
	26	#ifdef MAPCACHE_DEBUG
	27	# define DPRINTF(fmt, ...) do { \
	28	fprintf(stderr, "xen_mapcache: " fmt, ## __VA_ARGS__); \
	29	} while (0)
	30	#else
	31	# define DPRINTF(fmt, ...) do { } while (0)
	32	#endif
	33
	34	#if defined(__i386__)
	35	# define MCACHE_BUCKET_SHIFT 16
ea6c5f8f	36	# define MCACHE_MAX_SIZE (1UL<<31) /* 2GB Cap */
432d268c JN	37	#elif defined(__x86_64__)
432d268c JN	38	# define MCACHE_BUCKET_SHIFT 20
ea6c5f8f	39	# define MCACHE_MAX_SIZE (1UL<<35) /* 32GB Cap */
432d268c JN	40	#endif
	41	#define MCACHE_BUCKET_SIZE (1UL << MCACHE_BUCKET_SHIFT)
	42
e41d7c69 JK	43	#define mapcache_lock() ((void)0)
	44	#define mapcache_unlock() ((void)0)
	45
432d268c JN	46	typedef struct MapCacheEntry {
	47	target_phys_addr_t paddr_index;
	48	uint8_t *vaddr_base;
c13390cd	49	unsigned long *valid_mapping;
432d268c	50	uint8_t lock;
c13390cd	51	target_phys_addr_t size;
432d268c JN	52	struct MapCacheEntry *next;
	53	} MapCacheEntry;
	54
	55	typedef struct MapCacheRev {
	56	uint8_t *vaddr_req;
	57	target_phys_addr_t paddr_index;
c13390cd	58	target_phys_addr_t size;
432d268c JN	59	QTAILQ_ENTRY(MapCacheRev) next;
	60	} MapCacheRev;
	61
	62	typedef struct MapCache {
	63	MapCacheEntry *entry;
	64	unsigned long nr_buckets;
	65	QTAILQ_HEAD(map_cache_head, MapCacheRev) locked_entries;
	66
	67	/* For most cases (>99.9%), the page address is the same. */
	68	target_phys_addr_t last_address_index;
	69	uint8_t *last_address_vaddr;
	70	unsigned long max_mcache_size;
	71	unsigned int mcache_bucket_shift;
	72	} MapCache;
	73
	74	static MapCache *mapcache;
	75
c13390cd SS	76	static inline int test_bits(int nr, int size, const unsigned long *addr)
	77	{
	78	unsigned long res = find_next_zero_bit(addr, size + nr, nr);
	79	if (res >= nr + size)
	80	return 1;
	81	else
	82	return 0;
	83	}
	84
e41d7c69	85	void xen_map_cache_init(void)
432d268c JN	86	{
	87	unsigned long size;
	88	struct rlimit rlimit_as;
	89
	90	mapcache = qemu_mallocz(sizeof (MapCache));
	91
	92	QTAILQ_INIT(&mapcache->locked_entries);
	93	mapcache->last_address_index = -1;
	94
	95	getrlimit(RLIMIT_AS, &rlimit_as);
ea6c5f8f JB	96	if (rlimit_as.rlim_max < MCACHE_MAX_SIZE) {
	97	rlimit_as.rlim_cur = rlimit_as.rlim_max;
	98	} else {
	99	rlimit_as.rlim_cur = MCACHE_MAX_SIZE;
	100	}
	101
432d268c	102	setrlimit(RLIMIT_AS, &rlimit_as);
ea6c5f8f	103	mapcache->max_mcache_size = rlimit_as.rlim_cur;
432d268c JN	104
	105	mapcache->nr_buckets =
	106	(((mapcache->max_mcache_size >> XC_PAGE_SHIFT) +
	107	(1UL << (MCACHE_BUCKET_SHIFT - XC_PAGE_SHIFT)) - 1) >>
	108	(MCACHE_BUCKET_SHIFT - XC_PAGE_SHIFT));
	109
	110	size = mapcache->nr_buckets * sizeof (MapCacheEntry);
	111	size = (size + XC_PAGE_SIZE - 1) & ~(XC_PAGE_SIZE - 1);
e41d7c69 JK	112	DPRINTF("%s, nr_buckets = %lx size %lu\n", __func__,
e41d7c69 JK	113	mapcache->nr_buckets, size);
432d268c JN	114	mapcache->entry = qemu_mallocz(size);
	115	}
	116
e41d7c69 JK	117	static void xen_remap_bucket(MapCacheEntry *entry,
	118	target_phys_addr_t size,
	119	target_phys_addr_t address_index)
432d268c JN	120	{
	121	uint8_t *vaddr_base;
	122	xen_pfn_t *pfns;
	123	int *err;
ea6c5f8f	124	unsigned int i;
432d268c JN	125	target_phys_addr_t nb_pfn = size >> XC_PAGE_SHIFT;
432d268c JN	126
e41d7c69	127	trace_xen_remap_bucket(address_index);
432d268c JN	128
	129	pfns = qemu_mallocz(nb_pfn * sizeof (xen_pfn_t));
	130	err = qemu_mallocz(nb_pfn * sizeof (int));
	131
	132	if (entry->vaddr_base != NULL) {
c13390cd	133	if (munmap(entry->vaddr_base, entry->size) != 0) {
432d268c JN	134	perror("unmap fails");
	135	exit(-1);
	136	}
	137	}
c13390cd SS	138	if (entry->valid_mapping != NULL) {
	139	qemu_free(entry->valid_mapping);
	140	entry->valid_mapping = NULL;
	141	}
432d268c JN	142
	143	for (i = 0; i < nb_pfn; i++) {
	144	pfns[i] = (address_index << (MCACHE_BUCKET_SHIFT-XC_PAGE_SHIFT)) + i;
	145	}
	146
	147	vaddr_base = xc_map_foreign_bulk(xen_xc, xen_domid, PROT_READ\|PROT_WRITE,
	148	pfns, err, nb_pfn);
	149	if (vaddr_base == NULL) {
	150	perror("xc_map_foreign_bulk");
	151	exit(-1);
	152	}
	153
	154	entry->vaddr_base = vaddr_base;
	155	entry->paddr_index = address_index;
c13390cd SS	156	entry->size = size;
	157	entry->valid_mapping = (unsigned long ) qemu_mallocz(sizeof(unsigned long)
	158	BITS_TO_LONGS(size >> XC_PAGE_SHIFT));
432d268c	159
ea6c5f8f JB	160	bitmap_zero(entry->valid_mapping, nb_pfn);
	161	for (i = 0; i < nb_pfn; i++) {
	162	if (!err[i]) {
	163	bitmap_set(entry->valid_mapping, i, 1);
432d268c	164	}
432d268c JN	165	}
	166
	167	qemu_free(pfns);
	168	qemu_free(err);
	169	}
	170
e41d7c69 JK	171	uint8_t *xen_map_cache(target_phys_addr_t phys_addr, target_phys_addr_t size,
e41d7c69 JK	172	uint8_t lock)
432d268c JN	173	{
	174	MapCacheEntry entry, pentry = NULL;
	175	target_phys_addr_t address_index = phys_addr >> MCACHE_BUCKET_SHIFT;
	176	target_phys_addr_t address_offset = phys_addr & (MCACHE_BUCKET_SIZE - 1);
c13390cd	177	target_phys_addr_t __size = size;
432d268c	178
e41d7c69	179	trace_xen_map_cache(phys_addr);
432d268c	180
c13390cd	181	if (address_index == mapcache->last_address_index && !lock && !__size) {
e41d7c69	182	trace_xen_map_cache_return(mapcache->last_address_vaddr + address_offset);
432d268c JN	183	return mapcache->last_address_vaddr + address_offset;
	184	}
	185
c13390cd SS	186	/* size is always a multiple of MCACHE_BUCKET_SIZE */
	187	if ((address_offset + (__size % MCACHE_BUCKET_SIZE)) > MCACHE_BUCKET_SIZE)
	188	__size += MCACHE_BUCKET_SIZE;
	189	if (__size % MCACHE_BUCKET_SIZE)
	190	__size += MCACHE_BUCKET_SIZE - (__size % MCACHE_BUCKET_SIZE);
	191	if (!__size)
	192	__size = MCACHE_BUCKET_SIZE;
	193
432d268c JN	194	entry = &mapcache->entry[address_index % mapcache->nr_buckets];
432d268c JN	195
c13390cd SS	196	while (entry && entry->lock && entry->vaddr_base &&
	197	(entry->paddr_index != address_index \|\| entry->size != __size \|\|
	198	!test_bits(address_offset >> XC_PAGE_SHIFT, size >> XC_PAGE_SHIFT,
	199	entry->valid_mapping))) {
432d268c JN	200	pentry = entry;
	201	entry = entry->next;
	202	}
	203	if (!entry) {
	204	entry = qemu_mallocz(sizeof (MapCacheEntry));
	205	pentry->next = entry;
e41d7c69	206	xen_remap_bucket(entry, __size, address_index);
432d268c JN	207	} else if (!entry->lock) {
432d268c JN	208	if (!entry->vaddr_base \|\| entry->paddr_index != address_index \|\|
c13390cd SS	209	entry->size != __size \|\|
	210	!test_bits(address_offset >> XC_PAGE_SHIFT, size >> XC_PAGE_SHIFT,
	211	entry->valid_mapping)) {
e41d7c69	212	xen_remap_bucket(entry, __size, address_index);
432d268c JN	213	}
	214	}
	215
c13390cd SS	216	if(!test_bits(address_offset >> XC_PAGE_SHIFT, size >> XC_PAGE_SHIFT,
c13390cd SS	217	entry->valid_mapping)) {
432d268c	218	mapcache->last_address_index = -1;
e41d7c69	219	trace_xen_map_cache_return(NULL);
432d268c JN	220	return NULL;
	221	}
	222
	223	mapcache->last_address_index = address_index;
	224	mapcache->last_address_vaddr = entry->vaddr_base;
	225	if (lock) {
	226	MapCacheRev *reventry = qemu_mallocz(sizeof(MapCacheRev));
	227	entry->lock++;
	228	reventry->vaddr_req = mapcache->last_address_vaddr + address_offset;
	229	reventry->paddr_index = mapcache->last_address_index;
c13390cd	230	reventry->size = entry->size;
432d268c JN	231	QTAILQ_INSERT_HEAD(&mapcache->locked_entries, reventry, next);
	232	}
	233
e41d7c69	234	trace_xen_map_cache_return(mapcache->last_address_vaddr + address_offset);
432d268c JN	235	return mapcache->last_address_vaddr + address_offset;
	236	}
	237
e41d7c69	238	ram_addr_t xen_ram_addr_from_mapcache(void *ptr)
432d268c	239	{
c13390cd	240	MapCacheEntry entry = NULL, pentry = NULL;
432d268c JN	241	MapCacheRev *reventry;
432d268c JN	242	target_phys_addr_t paddr_index;
c13390cd	243	target_phys_addr_t size;
432d268c JN	244	int found = 0;
	245
	246	QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
	247	if (reventry->vaddr_req == ptr) {
	248	paddr_index = reventry->paddr_index;
c13390cd	249	size = reventry->size;
432d268c JN	250	found = 1;
	251	break;
	252	}
	253	}
	254	if (!found) {
e41d7c69	255	fprintf(stderr, "%s, could not find %p\n", __func__, ptr);
432d268c JN	256	QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
	257	DPRINTF(" "TARGET_FMT_plx" -> %p is present\n", reventry->paddr_index,
	258	reventry->vaddr_req);
	259	}
	260	abort();
	261	return 0;
	262	}
	263
c13390cd SS	264	entry = &mapcache->entry[paddr_index % mapcache->nr_buckets];
	265	while (entry && (entry->paddr_index != paddr_index \|\| entry->size != size)) {
	266	pentry = entry;
	267	entry = entry->next;
	268	}
	269	if (!entry) {
	270	DPRINTF("Trying to find address %p that is not in the mapcache!\n", ptr);
	271	return 0;
	272	}
	273	return (reventry->paddr_index << MCACHE_BUCKET_SHIFT) +
	274	((unsigned long) ptr - (unsigned long) entry->vaddr_base);
432d268c JN	275	}
432d268c JN	276
e41d7c69	277	void xen_invalidate_map_cache_entry(uint8_t *buffer)
432d268c JN	278	{
	279	MapCacheEntry entry = NULL, pentry = NULL;
	280	MapCacheRev *reventry;
	281	target_phys_addr_t paddr_index;
c13390cd	282	target_phys_addr_t size;
432d268c JN	283	int found = 0;
	284
	285	if (mapcache->last_address_vaddr == buffer) {
	286	mapcache->last_address_index = -1;
	287	}
	288
	289	QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
	290	if (reventry->vaddr_req == buffer) {
	291	paddr_index = reventry->paddr_index;
c13390cd	292	size = reventry->size;
432d268c JN	293	found = 1;
	294	break;
	295	}
	296	}
	297	if (!found) {
e41d7c69	298	DPRINTF("%s, could not find %p\n", __func__, buffer);
432d268c JN	299	QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
	300	DPRINTF(" "TARGET_FMT_plx" -> %p is present\n", reventry->paddr_index, reventry->vaddr_req);
	301	}
	302	return;
	303	}
	304	QTAILQ_REMOVE(&mapcache->locked_entries, reventry, next);
	305	qemu_free(reventry);
	306
	307	entry = &mapcache->entry[paddr_index % mapcache->nr_buckets];
c13390cd	308	while (entry && (entry->paddr_index != paddr_index \|\| entry->size != size)) {
432d268c JN	309	pentry = entry;
	310	entry = entry->next;
	311	}
	312	if (!entry) {
	313	DPRINTF("Trying to unmap address %p that is not in the mapcache!\n", buffer);
	314	return;
	315	}
	316	entry->lock--;
	317	if (entry->lock > 0 \|\| pentry == NULL) {
	318	return;
	319	}
	320
	321	pentry->next = entry->next;
c13390cd	322	if (munmap(entry->vaddr_base, entry->size) != 0) {
432d268c JN	323	perror("unmap fails");
	324	exit(-1);
	325	}
c13390cd	326	qemu_free(entry->valid_mapping);
432d268c JN	327	qemu_free(entry);
	328	}
	329
e41d7c69	330	void xen_invalidate_map_cache(void)
432d268c JN	331	{
	332	unsigned long i;
	333	MapCacheRev *reventry;
	334
	335	/* Flush pending AIO before destroying the mapcache */
	336	qemu_aio_flush();
	337
	338	QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
	339	DPRINTF("There should be no locked mappings at this time, "
	340	"but "TARGET_FMT_plx" -> %p is present\n",
	341	reventry->paddr_index, reventry->vaddr_req);
	342	}
	343
	344	mapcache_lock();
	345
	346	for (i = 0; i < mapcache->nr_buckets; i++) {
	347	MapCacheEntry *entry = &mapcache->entry[i];
	348
	349	if (entry->vaddr_base == NULL) {
	350	continue;
	351	}
	352
c13390cd	353	if (munmap(entry->vaddr_base, entry->size) != 0) {
432d268c JN	354	perror("unmap fails");
	355	exit(-1);
	356	}
	357
	358	entry->paddr_index = 0;
	359	entry->vaddr_base = NULL;
c13390cd SS	360	entry->size = 0;
	361	qemu_free(entry->valid_mapping);
	362	entry->valid_mapping = NULL;
432d268c JN	363	}
	364
	365	mapcache->last_address_index = -1;
	366	mapcache->last_address_vaddr = NULL;
	367
	368	mapcache_unlock();
	369	}