[qemu.git] / xen-mapcache.c

/*
 * Copyright (C) 2011       Citrix Ltd.
 *
 * This work is licensed under the terms of the GNU GPL, version 2.  See
 * the COPYING file in the top-level directory.
 *
 */

#include "config.h"

#include <sys/resource.h>

#include "hw/xen_backend.h"
#include "blockdev.h"
#include "bitmap.h"

#include <xen/hvm/params.h>
#include <sys/mman.h>

#include "xen-mapcache.h"
#include "trace.h"


//#define MAPCACHE_DEBUG

#ifdef MAPCACHE_DEBUG
#  define DPRINTF(fmt, ...) do { \
    fprintf(stderr, "xen_mapcache: " fmt, ## __VA_ARGS__); \
} while (0)
#else
#  define DPRINTF(fmt, ...) do { } while (0)
#endif

#if defined(__i386__)
#  define MCACHE_BUCKET_SHIFT 16
#  define MCACHE_MAX_SIZE     (1UL<<31) /* 2GB Cap */
#elif defined(__x86_64__)
#  define MCACHE_BUCKET_SHIFT 20
#  define MCACHE_MAX_SIZE     (1UL<<35) /* 32GB Cap */
#endif
#define MCACHE_BUCKET_SIZE (1UL << MCACHE_BUCKET_SHIFT)

/* This is the size of the virtual address space reserve to QEMU that will not
 * be use by MapCache.
 * From empirical tests I observed that qemu use 75MB more than the
 * max_mcache_size.
 */
#define NON_MCACHE_MEMORY_SIZE (80 * 1024 * 1024)

#define mapcache_lock()   ((void)0)
#define mapcache_unlock() ((void)0)

typedef struct MapCacheEntry {
    target_phys_addr_t paddr_index;
    uint8_t *vaddr_base;
    unsigned long *valid_mapping;
    uint8_t lock;
    target_phys_addr_t size;
    struct MapCacheEntry *next;
} MapCacheEntry;

typedef struct MapCacheRev {
    uint8_t *vaddr_req;
    target_phys_addr_t paddr_index;
    target_phys_addr_t size;
    QTAILQ_ENTRY(MapCacheRev) next;
} MapCacheRev;

typedef struct MapCache {
    MapCacheEntry *entry;
    unsigned long nr_buckets;
    QTAILQ_HEAD(map_cache_head, MapCacheRev) locked_entries;

    /* For most cases (>99.9%), the page address is the same. */
    target_phys_addr_t last_address_index;
    uint8_t *last_address_vaddr;
    unsigned long max_mcache_size;
    unsigned int mcache_bucket_shift;
} MapCache;

static MapCache *mapcache;

static inline int test_bits(int nr, int size, const unsigned long *addr)
{
    unsigned long res = find_next_zero_bit(addr, size + nr, nr);
    if (res >= nr + size)
        return 1;
    else
        return 0;
}

void xen_map_cache_init(void)
{
    unsigned long size;
    struct rlimit rlimit_as;

    mapcache = g_malloc0(sizeof (MapCache));

    QTAILQ_INIT(&mapcache->locked_entries);
    mapcache->last_address_index = -1;

    if (geteuid() == 0) {
        rlimit_as.rlim_cur = RLIM_INFINITY;
        rlimit_as.rlim_max = RLIM_INFINITY;
        mapcache->max_mcache_size = MCACHE_MAX_SIZE;
    } else {
        getrlimit(RLIMIT_AS, &rlimit_as);
        rlimit_as.rlim_cur = rlimit_as.rlim_max;

        if (rlimit_as.rlim_max != RLIM_INFINITY) {
            fprintf(stderr, "Warning: QEMU's maximum size of virtual"
                    " memory is not infinity.\n");
        }
        if (rlimit_as.rlim_max < MCACHE_MAX_SIZE + NON_MCACHE_MEMORY_SIZE) {
            mapcache->max_mcache_size = rlimit_as.rlim_max -
                NON_MCACHE_MEMORY_SIZE;
        } else {
            mapcache->max_mcache_size = MCACHE_MAX_SIZE;
        }
    }

    setrlimit(RLIMIT_AS, &rlimit_as);

    mapcache->nr_buckets =
        (((mapcache->max_mcache_size >> XC_PAGE_SHIFT) +
          (1UL << (MCACHE_BUCKET_SHIFT - XC_PAGE_SHIFT)) - 1) >>
         (MCACHE_BUCKET_SHIFT - XC_PAGE_SHIFT));

    size = mapcache->nr_buckets * sizeof (MapCacheEntry);
    size = (size + XC_PAGE_SIZE - 1) & ~(XC_PAGE_SIZE - 1);
    DPRINTF("%s, nr_buckets = %lx size %lu\n", __func__,
            mapcache->nr_buckets, size);
    mapcache->entry = g_malloc0(size);
}

static void xen_remap_bucket(MapCacheEntry *entry,
                             target_phys_addr_t size,
                             target_phys_addr_t address_index)
{
    uint8_t *vaddr_base;
    xen_pfn_t *pfns;
    int *err;
    unsigned int i;
    target_phys_addr_t nb_pfn = size >> XC_PAGE_SHIFT;

    trace_xen_remap_bucket(address_index);

    pfns = g_malloc0(nb_pfn * sizeof (xen_pfn_t));
    err = g_malloc0(nb_pfn * sizeof (int));

    if (entry->vaddr_base != NULL) {
        if (munmap(entry->vaddr_base, entry->size) != 0) {
            perror("unmap fails");
            exit(-1);
        }
    }
    if (entry->valid_mapping != NULL) {
        g_free(entry->valid_mapping);
        entry->valid_mapping = NULL;
    }

    for (i = 0; i < nb_pfn; i++) {
        pfns[i] = (address_index << (MCACHE_BUCKET_SHIFT-XC_PAGE_SHIFT)) + i;
    }

    vaddr_base = xc_map_foreign_bulk(xen_xc, xen_domid, PROT_READ|PROT_WRITE,
                                     pfns, err, nb_pfn);
    if (vaddr_base == NULL) {
        perror("xc_map_foreign_bulk");
        exit(-1);
    }

    entry->vaddr_base = vaddr_base;
    entry->paddr_index = address_index;
    entry->size = size;
    entry->valid_mapping = (unsigned long *) g_malloc0(sizeof(unsigned long) *
            BITS_TO_LONGS(size >> XC_PAGE_SHIFT));

    bitmap_zero(entry->valid_mapping, nb_pfn);
    for (i = 0; i < nb_pfn; i++) {
        if (!err[i]) {
            bitmap_set(entry->valid_mapping, i, 1);
        }
    }

    g_free(pfns);
    g_free(err);
}

uint8_t *xen_map_cache(target_phys_addr_t phys_addr, target_phys_addr_t size,
                       uint8_t lock)
{
    MapCacheEntry *entry, *pentry = NULL;
    target_phys_addr_t address_index  = phys_addr >> MCACHE_BUCKET_SHIFT;
    target_phys_addr_t address_offset = phys_addr & (MCACHE_BUCKET_SIZE - 1);
    target_phys_addr_t __size = size;

    trace_xen_map_cache(phys_addr);

    if (address_index == mapcache->last_address_index && !lock && !__size) {
        trace_xen_map_cache_return(mapcache->last_address_vaddr + address_offset);
        return mapcache->last_address_vaddr + address_offset;
    }

    /* size is always a multiple of MCACHE_BUCKET_SIZE */
    if ((address_offset + (__size % MCACHE_BUCKET_SIZE)) > MCACHE_BUCKET_SIZE)
        __size += MCACHE_BUCKET_SIZE;
    if (__size % MCACHE_BUCKET_SIZE)
        __size += MCACHE_BUCKET_SIZE - (__size % MCACHE_BUCKET_SIZE);
    if (!__size)
        __size = MCACHE_BUCKET_SIZE;

    entry = &mapcache->entry[address_index % mapcache->nr_buckets];

    while (entry && entry->lock && entry->vaddr_base &&
            (entry->paddr_index != address_index || entry->size != __size ||
             !test_bits(address_offset >> XC_PAGE_SHIFT, size >> XC_PAGE_SHIFT,
                 entry->valid_mapping))) {
        pentry = entry;
        entry = entry->next;
    }
    if (!entry) {
        entry = g_malloc0(sizeof (MapCacheEntry));
        pentry->next = entry;
        xen_remap_bucket(entry, __size, address_index);
    } else if (!entry->lock) {
        if (!entry->vaddr_base || entry->paddr_index != address_index ||
                entry->size != __size ||
                !test_bits(address_offset >> XC_PAGE_SHIFT, size >> XC_PAGE_SHIFT,
                    entry->valid_mapping)) {
            xen_remap_bucket(entry, __size, address_index);
        }
    }

    if(!test_bits(address_offset >> XC_PAGE_SHIFT, size >> XC_PAGE_SHIFT,
                entry->valid_mapping)) {
        mapcache->last_address_index = -1;
        trace_xen_map_cache_return(NULL);
        return NULL;
    }

    mapcache->last_address_index = address_index;
    mapcache->last_address_vaddr = entry->vaddr_base;
    if (lock) {
        MapCacheRev *reventry = g_malloc0(sizeof(MapCacheRev));
        entry->lock++;
        reventry->vaddr_req = mapcache->last_address_vaddr + address_offset;
        reventry->paddr_index = mapcache->last_address_index;
        reventry->size = entry->size;
        QTAILQ_INSERT_HEAD(&mapcache->locked_entries, reventry, next);
    }

    trace_xen_map_cache_return(mapcache->last_address_vaddr + address_offset);
    return mapcache->last_address_vaddr + address_offset;
}

ram_addr_t xen_ram_addr_from_mapcache(void *ptr)
{
    MapCacheEntry *entry = NULL;
    MapCacheRev *reventry;
    target_phys_addr_t paddr_index;
    target_phys_addr_t size;
    int found = 0;

    QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
        if (reventry->vaddr_req == ptr) {
            paddr_index = reventry->paddr_index;
            size = reventry->size;
            found = 1;
            break;
        }
    }
    if (!found) {
        fprintf(stderr, "%s, could not find %p\n", __func__, ptr);
        QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
            DPRINTF("   "TARGET_FMT_plx" -> %p is present\n", reventry->paddr_index,
                    reventry->vaddr_req);
        }
        abort();
        return 0;
    }

    entry = &mapcache->entry[paddr_index % mapcache->nr_buckets];
    while (entry && (entry->paddr_index != paddr_index || entry->size != size)) {
        entry = entry->next;
    }
    if (!entry) {
        DPRINTF("Trying to find address %p that is not in the mapcache!\n", ptr);
        return 0;
    }
    return (reventry->paddr_index << MCACHE_BUCKET_SHIFT) +
        ((unsigned long) ptr - (unsigned long) entry->vaddr_base);
}

void xen_invalidate_map_cache_entry(uint8_t *buffer)
{
    MapCacheEntry *entry = NULL, *pentry = NULL;
    MapCacheRev *reventry;
    target_phys_addr_t paddr_index;
    target_phys_addr_t size;
    int found = 0;

    if (mapcache->last_address_vaddr == buffer) {
        mapcache->last_address_index = -1;
    }

    QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
        if (reventry->vaddr_req == buffer) {
            paddr_index = reventry->paddr_index;
            size = reventry->size;
            found = 1;
            break;
        }
    }
    if (!found) {
        DPRINTF("%s, could not find %p\n", __func__, buffer);
        QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
            DPRINTF("   "TARGET_FMT_plx" -> %p is present\n", reventry->paddr_index, reventry->vaddr_req);
        }
        return;
    }
    QTAILQ_REMOVE(&mapcache->locked_entries, reventry, next);
    g_free(reventry);

    entry = &mapcache->entry[paddr_index % mapcache->nr_buckets];
    while (entry && (entry->paddr_index != paddr_index || entry->size != size)) {
        pentry = entry;
        entry = entry->next;
    }
    if (!entry) {
        DPRINTF("Trying to unmap address %p that is not in the mapcache!\n", buffer);
        return;
    }
    entry->lock--;
    if (entry->lock > 0 || pentry == NULL) {
        return;
    }

    pentry->next = entry->next;
    if (munmap(entry->vaddr_base, entry->size) != 0) {
        perror("unmap fails");
        exit(-1);
    }
    g_free(entry->valid_mapping);
    g_free(entry);
}

void xen_invalidate_map_cache(void)
{
    unsigned long i;
    MapCacheRev *reventry;

    /* Flush pending AIO before destroying the mapcache */
    qemu_aio_flush();

    QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
        DPRINTF("There should be no locked mappings at this time, "
                "but "TARGET_FMT_plx" -> %p is present\n",
                reventry->paddr_index, reventry->vaddr_req);
    }

    mapcache_lock();

    for (i = 0; i < mapcache->nr_buckets; i++) {
        MapCacheEntry *entry = &mapcache->entry[i];

        if (entry->vaddr_base == NULL) {
            continue;
        }

        if (munmap(entry->vaddr_base, entry->size) != 0) {
            perror("unmap fails");
            exit(-1);
        }

        entry->paddr_index = 0;
        entry->vaddr_base = NULL;
        entry->size = 0;
        g_free(entry->valid_mapping);
        entry->valid_mapping = NULL;
    }

    mapcache->last_address_index = -1;
    mapcache->last_address_vaddr = NULL;

    mapcache_unlock();
}
Commit	Line	Data
432d268c JN	1	/*
	2	* Copyright (C) 2011 Citrix Ltd.
	3	*
	4	* This work is licensed under the terms of the GNU GPL, version 2. See
	5	* the COPYING file in the top-level directory.
	6	*
	7	*/
	8
	9	#include "config.h"
	10
	11	#include <sys/resource.h>
	12
	13	#include "hw/xen_backend.h"
	14	#include "blockdev.h"
ea6c5f8f	15	#include "bitmap.h"
432d268c JN	16
	17	#include <xen/hvm/params.h>
	18	#include <sys/mman.h>
	19
	20	#include "xen-mapcache.h"
	21	#include "trace.h"
	22
	23
	24	//#define MAPCACHE_DEBUG
	25
	26	#ifdef MAPCACHE_DEBUG
	27	# define DPRINTF(fmt, ...) do { \
	28	fprintf(stderr, "xen_mapcache: " fmt, ## __VA_ARGS__); \
	29	} while (0)
	30	#else
	31	# define DPRINTF(fmt, ...) do { } while (0)
	32	#endif
	33
	34	#if defined(__i386__)
	35	# define MCACHE_BUCKET_SHIFT 16
ea6c5f8f	36	# define MCACHE_MAX_SIZE (1UL<<31) /* 2GB Cap */
432d268c JN	37	#elif defined(__x86_64__)
432d268c JN	38	# define MCACHE_BUCKET_SHIFT 20
ea6c5f8f	39	# define MCACHE_MAX_SIZE (1UL<<35) /* 32GB Cap */
432d268c JN	40	#endif
	41	#define MCACHE_BUCKET_SIZE (1UL << MCACHE_BUCKET_SHIFT)
	42
56c119e5 AP	43	/* This is the size of the virtual address space reserve to QEMU that will not
	44	* be use by MapCache.
	45	* From empirical tests I observed that qemu use 75MB more than the
	46	* max_mcache_size.
	47	*/
	48	#define NON_MCACHE_MEMORY_SIZE (80 * 1024 * 1024)
	49
e41d7c69 JK	50	#define mapcache_lock() ((void)0)
	51	#define mapcache_unlock() ((void)0)
	52
432d268c JN	53	typedef struct MapCacheEntry {
	54	target_phys_addr_t paddr_index;
	55	uint8_t *vaddr_base;
c13390cd	56	unsigned long *valid_mapping;
432d268c	57	uint8_t lock;
c13390cd	58	target_phys_addr_t size;
432d268c JN	59	struct MapCacheEntry *next;
	60	} MapCacheEntry;
	61
	62	typedef struct MapCacheRev {
	63	uint8_t *vaddr_req;
	64	target_phys_addr_t paddr_index;
c13390cd	65	target_phys_addr_t size;
432d268c JN	66	QTAILQ_ENTRY(MapCacheRev) next;
	67	} MapCacheRev;
	68
	69	typedef struct MapCache {
	70	MapCacheEntry *entry;
	71	unsigned long nr_buckets;
	72	QTAILQ_HEAD(map_cache_head, MapCacheRev) locked_entries;
	73
	74	/* For most cases (>99.9%), the page address is the same. */
	75	target_phys_addr_t last_address_index;
	76	uint8_t *last_address_vaddr;
	77	unsigned long max_mcache_size;
	78	unsigned int mcache_bucket_shift;
	79	} MapCache;
	80
	81	static MapCache *mapcache;
	82
c13390cd SS	83	static inline int test_bits(int nr, int size, const unsigned long *addr)
	84	{
	85	unsigned long res = find_next_zero_bit(addr, size + nr, nr);
	86	if (res >= nr + size)
	87	return 1;
	88	else
	89	return 0;
	90	}
	91
e41d7c69	92	void xen_map_cache_init(void)
432d268c JN	93	{
	94	unsigned long size;
	95	struct rlimit rlimit_as;
	96
7267c094	97	mapcache = g_malloc0(sizeof (MapCache));
432d268c JN	98
	99	QTAILQ_INIT(&mapcache->locked_entries);
	100	mapcache->last_address_index = -1;
	101
56c119e5 AP	102	if (geteuid() == 0) {
	103	rlimit_as.rlim_cur = RLIM_INFINITY;
	104	rlimit_as.rlim_max = RLIM_INFINITY;
	105	mapcache->max_mcache_size = MCACHE_MAX_SIZE;
ea6c5f8f	106	} else {
56c119e5 AP	107	getrlimit(RLIMIT_AS, &rlimit_as);
	108	rlimit_as.rlim_cur = rlimit_as.rlim_max;
	109
	110	if (rlimit_as.rlim_max != RLIM_INFINITY) {
	111	fprintf(stderr, "Warning: QEMU's maximum size of virtual"
	112	" memory is not infinity.\n");
	113	}
	114	if (rlimit_as.rlim_max < MCACHE_MAX_SIZE + NON_MCACHE_MEMORY_SIZE) {
	115	mapcache->max_mcache_size = rlimit_as.rlim_max -
	116	NON_MCACHE_MEMORY_SIZE;
	117	} else {
	118	mapcache->max_mcache_size = MCACHE_MAX_SIZE;
	119	}
ea6c5f8f JB	120	}
ea6c5f8f JB	121
432d268c	122	setrlimit(RLIMIT_AS, &rlimit_as);
432d268c JN	123
	124	mapcache->nr_buckets =
	125	(((mapcache->max_mcache_size >> XC_PAGE_SHIFT) +
	126	(1UL << (MCACHE_BUCKET_SHIFT - XC_PAGE_SHIFT)) - 1) >>
	127	(MCACHE_BUCKET_SHIFT - XC_PAGE_SHIFT));
	128
	129	size = mapcache->nr_buckets * sizeof (MapCacheEntry);
	130	size = (size + XC_PAGE_SIZE - 1) & ~(XC_PAGE_SIZE - 1);
e41d7c69 JK	131	DPRINTF("%s, nr_buckets = %lx size %lu\n", __func__,
e41d7c69 JK	132	mapcache->nr_buckets, size);
7267c094	133	mapcache->entry = g_malloc0(size);
432d268c JN	134	}
432d268c JN	135
e41d7c69 JK	136	static void xen_remap_bucket(MapCacheEntry *entry,
	137	target_phys_addr_t size,
	138	target_phys_addr_t address_index)
432d268c JN	139	{
	140	uint8_t *vaddr_base;
	141	xen_pfn_t *pfns;
	142	int *err;
ea6c5f8f	143	unsigned int i;
432d268c JN	144	target_phys_addr_t nb_pfn = size >> XC_PAGE_SHIFT;
432d268c JN	145
e41d7c69	146	trace_xen_remap_bucket(address_index);
432d268c	147
7267c094 AL	148	pfns = g_malloc0(nb_pfn * sizeof (xen_pfn_t));
7267c094 AL	149	err = g_malloc0(nb_pfn * sizeof (int));
432d268c JN	150
432d268c JN	151	if (entry->vaddr_base != NULL) {
c13390cd	152	if (munmap(entry->vaddr_base, entry->size) != 0) {
432d268c JN	153	perror("unmap fails");
	154	exit(-1);
	155	}
	156	}
c13390cd	157	if (entry->valid_mapping != NULL) {
7267c094	158	g_free(entry->valid_mapping);
c13390cd SS	159	entry->valid_mapping = NULL;
c13390cd SS	160	}
432d268c JN	161
	162	for (i = 0; i < nb_pfn; i++) {
	163	pfns[i] = (address_index << (MCACHE_BUCKET_SHIFT-XC_PAGE_SHIFT)) + i;
	164	}
	165
	166	vaddr_base = xc_map_foreign_bulk(xen_xc, xen_domid, PROT_READ\|PROT_WRITE,
	167	pfns, err, nb_pfn);
	168	if (vaddr_base == NULL) {
	169	perror("xc_map_foreign_bulk");
	170	exit(-1);
	171	}
	172
	173	entry->vaddr_base = vaddr_base;
	174	entry->paddr_index = address_index;
c13390cd	175	entry->size = size;
7267c094	176	entry->valid_mapping = (unsigned long ) g_malloc0(sizeof(unsigned long)
c13390cd	177	BITS_TO_LONGS(size >> XC_PAGE_SHIFT));
432d268c	178
ea6c5f8f JB	179	bitmap_zero(entry->valid_mapping, nb_pfn);
	180	for (i = 0; i < nb_pfn; i++) {
	181	if (!err[i]) {
	182	bitmap_set(entry->valid_mapping, i, 1);
432d268c	183	}
432d268c JN	184	}
432d268c JN	185
7267c094 AL	186	g_free(pfns);
7267c094 AL	187	g_free(err);
432d268c JN	188	}
432d268c JN	189
e41d7c69 JK	190	uint8_t *xen_map_cache(target_phys_addr_t phys_addr, target_phys_addr_t size,
e41d7c69 JK	191	uint8_t lock)
432d268c JN	192	{
	193	MapCacheEntry entry, pentry = NULL;
	194	target_phys_addr_t address_index = phys_addr >> MCACHE_BUCKET_SHIFT;
	195	target_phys_addr_t address_offset = phys_addr & (MCACHE_BUCKET_SIZE - 1);
c13390cd	196	target_phys_addr_t __size = size;
432d268c	197
e41d7c69	198	trace_xen_map_cache(phys_addr);
432d268c	199
c13390cd	200	if (address_index == mapcache->last_address_index && !lock && !__size) {
e41d7c69	201	trace_xen_map_cache_return(mapcache->last_address_vaddr + address_offset);
432d268c JN	202	return mapcache->last_address_vaddr + address_offset;
	203	}
	204
c13390cd SS	205	/* size is always a multiple of MCACHE_BUCKET_SIZE */
	206	if ((address_offset + (__size % MCACHE_BUCKET_SIZE)) > MCACHE_BUCKET_SIZE)
	207	__size += MCACHE_BUCKET_SIZE;
	208	if (__size % MCACHE_BUCKET_SIZE)
	209	__size += MCACHE_BUCKET_SIZE - (__size % MCACHE_BUCKET_SIZE);
	210	if (!__size)
	211	__size = MCACHE_BUCKET_SIZE;
	212
432d268c JN	213	entry = &mapcache->entry[address_index % mapcache->nr_buckets];
432d268c JN	214
c13390cd SS	215	while (entry && entry->lock && entry->vaddr_base &&
	216	(entry->paddr_index != address_index \|\| entry->size != __size \|\|
	217	!test_bits(address_offset >> XC_PAGE_SHIFT, size >> XC_PAGE_SHIFT,
	218	entry->valid_mapping))) {
432d268c JN	219	pentry = entry;
	220	entry = entry->next;
	221	}
	222	if (!entry) {
7267c094	223	entry = g_malloc0(sizeof (MapCacheEntry));
432d268c	224	pentry->next = entry;
e41d7c69	225	xen_remap_bucket(entry, __size, address_index);
432d268c JN	226	} else if (!entry->lock) {
432d268c JN	227	if (!entry->vaddr_base \|\| entry->paddr_index != address_index \|\|
c13390cd SS	228	entry->size != __size \|\|
	229	!test_bits(address_offset >> XC_PAGE_SHIFT, size >> XC_PAGE_SHIFT,
	230	entry->valid_mapping)) {
e41d7c69	231	xen_remap_bucket(entry, __size, address_index);
432d268c JN	232	}
	233	}
	234
c13390cd SS	235	if(!test_bits(address_offset >> XC_PAGE_SHIFT, size >> XC_PAGE_SHIFT,
c13390cd SS	236	entry->valid_mapping)) {
432d268c	237	mapcache->last_address_index = -1;
e41d7c69	238	trace_xen_map_cache_return(NULL);
432d268c JN	239	return NULL;
	240	}
	241
	242	mapcache->last_address_index = address_index;
	243	mapcache->last_address_vaddr = entry->vaddr_base;
	244	if (lock) {
7267c094	245	MapCacheRev *reventry = g_malloc0(sizeof(MapCacheRev));
432d268c JN	246	entry->lock++;
	247	reventry->vaddr_req = mapcache->last_address_vaddr + address_offset;
	248	reventry->paddr_index = mapcache->last_address_index;
c13390cd	249	reventry->size = entry->size;
432d268c JN	250	QTAILQ_INSERT_HEAD(&mapcache->locked_entries, reventry, next);
	251	}
	252
e41d7c69	253	trace_xen_map_cache_return(mapcache->last_address_vaddr + address_offset);
432d268c JN	254	return mapcache->last_address_vaddr + address_offset;
	255	}
	256
e41d7c69	257	ram_addr_t xen_ram_addr_from_mapcache(void *ptr)
432d268c	258	{
ecf169b7	259	MapCacheEntry *entry = NULL;
432d268c JN	260	MapCacheRev *reventry;
432d268c JN	261	target_phys_addr_t paddr_index;
c13390cd	262	target_phys_addr_t size;
432d268c JN	263	int found = 0;
	264
	265	QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
	266	if (reventry->vaddr_req == ptr) {
	267	paddr_index = reventry->paddr_index;
c13390cd	268	size = reventry->size;
432d268c JN	269	found = 1;
	270	break;
	271	}
	272	}
	273	if (!found) {
e41d7c69	274	fprintf(stderr, "%s, could not find %p\n", __func__, ptr);
432d268c JN	275	QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
	276	DPRINTF(" "TARGET_FMT_plx" -> %p is present\n", reventry->paddr_index,
	277	reventry->vaddr_req);
	278	}
	279	abort();
	280	return 0;
	281	}
	282
c13390cd SS	283	entry = &mapcache->entry[paddr_index % mapcache->nr_buckets];
c13390cd SS	284	while (entry && (entry->paddr_index != paddr_index \|\| entry->size != size)) {
c13390cd SS	285	entry = entry->next;
	286	}
	287	if (!entry) {
	288	DPRINTF("Trying to find address %p that is not in the mapcache!\n", ptr);
	289	return 0;
	290	}
	291	return (reventry->paddr_index << MCACHE_BUCKET_SHIFT) +
	292	((unsigned long) ptr - (unsigned long) entry->vaddr_base);
432d268c JN	293	}
432d268c JN	294
e41d7c69	295	void xen_invalidate_map_cache_entry(uint8_t *buffer)
432d268c JN	296	{
	297	MapCacheEntry entry = NULL, pentry = NULL;
	298	MapCacheRev *reventry;
	299	target_phys_addr_t paddr_index;
c13390cd	300	target_phys_addr_t size;
432d268c JN	301	int found = 0;
	302
	303	if (mapcache->last_address_vaddr == buffer) {
	304	mapcache->last_address_index = -1;
	305	}
	306
	307	QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
	308	if (reventry->vaddr_req == buffer) {
	309	paddr_index = reventry->paddr_index;
c13390cd	310	size = reventry->size;
432d268c JN	311	found = 1;
	312	break;
	313	}
	314	}
	315	if (!found) {
e41d7c69	316	DPRINTF("%s, could not find %p\n", __func__, buffer);
432d268c JN	317	QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
	318	DPRINTF(" "TARGET_FMT_plx" -> %p is present\n", reventry->paddr_index, reventry->vaddr_req);
	319	}
	320	return;
	321	}
	322	QTAILQ_REMOVE(&mapcache->locked_entries, reventry, next);
7267c094	323	g_free(reventry);
432d268c JN	324
432d268c JN	325	entry = &mapcache->entry[paddr_index % mapcache->nr_buckets];
c13390cd	326	while (entry && (entry->paddr_index != paddr_index \|\| entry->size != size)) {
432d268c JN	327	pentry = entry;
	328	entry = entry->next;
	329	}
	330	if (!entry) {
	331	DPRINTF("Trying to unmap address %p that is not in the mapcache!\n", buffer);
	332	return;
	333	}
	334	entry->lock--;
	335	if (entry->lock > 0 \|\| pentry == NULL) {
	336	return;
	337	}
	338
	339	pentry->next = entry->next;
c13390cd	340	if (munmap(entry->vaddr_base, entry->size) != 0) {
432d268c JN	341	perror("unmap fails");
	342	exit(-1);
	343	}
7267c094 AL	344	g_free(entry->valid_mapping);
7267c094 AL	345	g_free(entry);
432d268c JN	346	}
432d268c JN	347
e41d7c69	348	void xen_invalidate_map_cache(void)
432d268c JN	349	{
	350	unsigned long i;
	351	MapCacheRev *reventry;
	352
	353	/* Flush pending AIO before destroying the mapcache */
	354	qemu_aio_flush();
	355
	356	QTAILQ_FOREACH(reventry, &mapcache->locked_entries, next) {
	357	DPRINTF("There should be no locked mappings at this time, "
	358	"but "TARGET_FMT_plx" -> %p is present\n",
	359	reventry->paddr_index, reventry->vaddr_req);
	360	}
	361
	362	mapcache_lock();
	363
	364	for (i = 0; i < mapcache->nr_buckets; i++) {
	365	MapCacheEntry *entry = &mapcache->entry[i];
	366
	367	if (entry->vaddr_base == NULL) {
	368	continue;
	369	}
	370
c13390cd	371	if (munmap(entry->vaddr_base, entry->size) != 0) {
432d268c JN	372	perror("unmap fails");
	373	exit(-1);
	374	}
	375
	376	entry->paddr_index = 0;
	377	entry->vaddr_base = NULL;
c13390cd	378	entry->size = 0;
7267c094	379	g_free(entry->valid_mapping);
c13390cd	380	entry->valid_mapping = NULL;
432d268c JN	381	}
	382
	383	mapcache->last_address_index = -1;
	384	mapcache->last_address_vaddr = NULL;
	385
	386	mapcache_unlock();
	387	}