2 * Copyright (C) 2011 Citrix Ltd.
4 * This work is licensed under the terms of the GNU GPL, version 2. See
5 * the COPYING file in the top-level directory.
7 * Contributions after 2012-01-13 are licensed under the terms of the
8 * GNU GPL, version 2 or (at your option) any later version.
11 #include "qemu/osdep.h"
12 #include "qemu/error-report.h"
14 #include <sys/resource.h>
16 #include "hw/xen/xen_backend.h"
17 #include "qemu/bitmap.h"
19 #include <xen/hvm/params.h>
21 #include "sysemu/xen-mapcache.h"
25 //#define MAPCACHE_DEBUG
28 # define DPRINTF(fmt, ...) do { \
29 fprintf(stderr, "xen_mapcache: " fmt, ## __VA_ARGS__); \
32 # define DPRINTF(fmt, ...) do { } while (0)
35 #if HOST_LONG_BITS == 32
36 # define MCACHE_BUCKET_SHIFT 16
37 # define MCACHE_MAX_SIZE (1UL<<31) /* 2GB Cap */
39 # define MCACHE_BUCKET_SHIFT 20
40 # define MCACHE_MAX_SIZE (1UL<<35) /* 32GB Cap */
42 #define MCACHE_BUCKET_SIZE (1UL << MCACHE_BUCKET_SHIFT)
44 /* This is the size of the virtual address space reserve to QEMU that will not
46 * From empirical tests I observed that qemu use 75MB more than the
49 #define NON_MCACHE_MEMORY_SIZE (80 * 1024 * 1024)
51 typedef struct MapCacheEntry
{
54 unsigned long *valid_mapping
;
56 #define XEN_MAPCACHE_ENTRY_DUMMY (1 << 0)
59 struct MapCacheEntry
*next
;
62 typedef struct MapCacheRev
{
66 QTAILQ_ENTRY(MapCacheRev
) next
;
70 typedef struct MapCache
{
72 unsigned long nr_buckets
;
73 QTAILQ_HEAD(map_cache_head
, MapCacheRev
) locked_entries
;
75 /* For most cases (>99.9%), the page address is the same. */
76 MapCacheEntry
*last_entry
;
77 unsigned long max_mcache_size
;
78 unsigned int mcache_bucket_shift
;
80 phys_offset_to_gaddr_t phys_offset_to_gaddr
;
85 static MapCache
*mapcache
;
87 static inline void mapcache_lock(void)
89 qemu_mutex_lock(&mapcache
->lock
);
92 static inline void mapcache_unlock(void)
94 qemu_mutex_unlock(&mapcache
->lock
);
97 static inline int test_bits(int nr
, int size
, const unsigned long *addr
)
99 unsigned long res
= find_next_zero_bit(addr
, size
+ nr
, nr
);
100 if (res
>= nr
+ size
)
106 void xen_map_cache_init(phys_offset_to_gaddr_t f
, void *opaque
)
109 struct rlimit rlimit_as
;
111 mapcache
= g_malloc0(sizeof (MapCache
));
113 mapcache
->phys_offset_to_gaddr
= f
;
114 mapcache
->opaque
= opaque
;
115 qemu_mutex_init(&mapcache
->lock
);
117 QTAILQ_INIT(&mapcache
->locked_entries
);
119 if (geteuid() == 0) {
120 rlimit_as
.rlim_cur
= RLIM_INFINITY
;
121 rlimit_as
.rlim_max
= RLIM_INFINITY
;
122 mapcache
->max_mcache_size
= MCACHE_MAX_SIZE
;
124 getrlimit(RLIMIT_AS
, &rlimit_as
);
125 rlimit_as
.rlim_cur
= rlimit_as
.rlim_max
;
127 if (rlimit_as
.rlim_max
!= RLIM_INFINITY
) {
128 warn_report("QEMU's maximum size of virtual"
129 " memory is not infinity");
131 if (rlimit_as
.rlim_max
< MCACHE_MAX_SIZE
+ NON_MCACHE_MEMORY_SIZE
) {
132 mapcache
->max_mcache_size
= rlimit_as
.rlim_max
-
133 NON_MCACHE_MEMORY_SIZE
;
135 mapcache
->max_mcache_size
= MCACHE_MAX_SIZE
;
139 setrlimit(RLIMIT_AS
, &rlimit_as
);
141 mapcache
->nr_buckets
=
142 (((mapcache
->max_mcache_size
>> XC_PAGE_SHIFT
) +
143 (1UL << (MCACHE_BUCKET_SHIFT
- XC_PAGE_SHIFT
)) - 1) >>
144 (MCACHE_BUCKET_SHIFT
- XC_PAGE_SHIFT
));
146 size
= mapcache
->nr_buckets
* sizeof (MapCacheEntry
);
147 size
= (size
+ XC_PAGE_SIZE
- 1) & ~(XC_PAGE_SIZE
- 1);
148 DPRINTF("%s, nr_buckets = %lx size %lu\n", __func__
,
149 mapcache
->nr_buckets
, size
);
150 mapcache
->entry
= g_malloc0(size
);
153 static void xen_remap_bucket(MapCacheEntry
*entry
,
156 hwaddr address_index
,
163 hwaddr nb_pfn
= size
>> XC_PAGE_SHIFT
;
165 trace_xen_remap_bucket(address_index
);
167 pfns
= g_malloc0(nb_pfn
* sizeof (xen_pfn_t
));
168 err
= g_malloc0(nb_pfn
* sizeof (int));
170 if (entry
->vaddr_base
!= NULL
) {
171 if (!(entry
->flags
& XEN_MAPCACHE_ENTRY_DUMMY
)) {
172 ram_block_notify_remove(entry
->vaddr_base
, entry
->size
);
174 if (munmap(entry
->vaddr_base
, entry
->size
) != 0) {
175 perror("unmap fails");
179 g_free(entry
->valid_mapping
);
180 entry
->valid_mapping
= NULL
;
182 for (i
= 0; i
< nb_pfn
; i
++) {
183 pfns
[i
] = (address_index
<< (MCACHE_BUCKET_SHIFT
-XC_PAGE_SHIFT
)) + i
;
187 vaddr_base
= xenforeignmemory_map2(xen_fmem
, xen_domid
, vaddr
,
188 PROT_READ
| PROT_WRITE
, 0,
190 if (vaddr_base
== NULL
) {
191 perror("xenforeignmemory_map2");
196 * We create dummy mappings where we are unable to create a foreign
197 * mapping immediately due to certain circumstances (i.e. on resume now)
199 vaddr_base
= mmap(vaddr
, size
, PROT_READ
| PROT_WRITE
,
200 MAP_ANON
| MAP_SHARED
, -1, 0);
201 if (vaddr_base
== MAP_FAILED
) {
207 if (!(entry
->flags
& XEN_MAPCACHE_ENTRY_DUMMY
)) {
208 ram_block_notify_add(vaddr_base
, size
);
211 entry
->vaddr_base
= vaddr_base
;
212 entry
->paddr_index
= address_index
;
214 entry
->valid_mapping
= (unsigned long *) g_malloc0(sizeof(unsigned long) *
215 BITS_TO_LONGS(size
>> XC_PAGE_SHIFT
));
218 entry
->flags
|= XEN_MAPCACHE_ENTRY_DUMMY
;
220 entry
->flags
&= ~(XEN_MAPCACHE_ENTRY_DUMMY
);
223 bitmap_zero(entry
->valid_mapping
, nb_pfn
);
224 for (i
= 0; i
< nb_pfn
; i
++) {
226 bitmap_set(entry
->valid_mapping
, i
, 1);
234 static uint8_t *xen_map_cache_unlocked(hwaddr phys_addr
, hwaddr size
,
235 uint8_t lock
, bool dma
)
237 MapCacheEntry
*entry
, *pentry
= NULL
,
238 *free_entry
= NULL
, *free_pentry
= NULL
;
239 hwaddr address_index
;
240 hwaddr address_offset
;
241 hwaddr cache_size
= size
;
242 hwaddr test_bit_size
;
243 bool translated G_GNUC_UNUSED
= false;
247 address_index
= phys_addr
>> MCACHE_BUCKET_SHIFT
;
248 address_offset
= phys_addr
& (MCACHE_BUCKET_SIZE
- 1);
250 trace_xen_map_cache(phys_addr
);
252 /* test_bit_size is always a multiple of XC_PAGE_SIZE */
254 test_bit_size
= size
+ (phys_addr
& (XC_PAGE_SIZE
- 1));
256 if (test_bit_size
% XC_PAGE_SIZE
) {
257 test_bit_size
+= XC_PAGE_SIZE
- (test_bit_size
% XC_PAGE_SIZE
);
260 test_bit_size
= XC_PAGE_SIZE
;
263 if (mapcache
->last_entry
!= NULL
&&
264 mapcache
->last_entry
->paddr_index
== address_index
&&
266 test_bits(address_offset
>> XC_PAGE_SHIFT
,
267 test_bit_size
>> XC_PAGE_SHIFT
,
268 mapcache
->last_entry
->valid_mapping
)) {
269 trace_xen_map_cache_return(mapcache
->last_entry
->vaddr_base
+ address_offset
);
270 return mapcache
->last_entry
->vaddr_base
+ address_offset
;
273 /* size is always a multiple of MCACHE_BUCKET_SIZE */
275 cache_size
= size
+ address_offset
;
276 if (cache_size
% MCACHE_BUCKET_SIZE
) {
277 cache_size
+= MCACHE_BUCKET_SIZE
- (cache_size
% MCACHE_BUCKET_SIZE
);
280 cache_size
= MCACHE_BUCKET_SIZE
;
283 entry
= &mapcache
->entry
[address_index
% mapcache
->nr_buckets
];
285 while (entry
&& (lock
|| entry
->lock
) && entry
->vaddr_base
&&
286 (entry
->paddr_index
!= address_index
|| entry
->size
!= cache_size
||
287 !test_bits(address_offset
>> XC_PAGE_SHIFT
,
288 test_bit_size
>> XC_PAGE_SHIFT
,
289 entry
->valid_mapping
))) {
290 if (!free_entry
&& !entry
->lock
) {
292 free_pentry
= pentry
;
297 if (!entry
&& free_entry
) {
299 pentry
= free_pentry
;
302 entry
= g_malloc0(sizeof (MapCacheEntry
));
303 pentry
->next
= entry
;
304 xen_remap_bucket(entry
, NULL
, cache_size
, address_index
, dummy
);
305 } else if (!entry
->lock
) {
306 if (!entry
->vaddr_base
|| entry
->paddr_index
!= address_index
||
307 entry
->size
!= cache_size
||
308 !test_bits(address_offset
>> XC_PAGE_SHIFT
,
309 test_bit_size
>> XC_PAGE_SHIFT
,
310 entry
->valid_mapping
)) {
311 xen_remap_bucket(entry
, NULL
, cache_size
, address_index
, dummy
);
315 if(!test_bits(address_offset
>> XC_PAGE_SHIFT
,
316 test_bit_size
>> XC_PAGE_SHIFT
,
317 entry
->valid_mapping
)) {
318 mapcache
->last_entry
= NULL
;
319 #ifdef XEN_COMPAT_PHYSMAP
320 if (!translated
&& mapcache
->phys_offset_to_gaddr
) {
321 phys_addr
= mapcache
->phys_offset_to_gaddr(phys_addr
, size
);
326 if (!dummy
&& runstate_check(RUN_STATE_INMIGRATE
)) {
330 trace_xen_map_cache_return(NULL
);
334 mapcache
->last_entry
= entry
;
336 MapCacheRev
*reventry
= g_malloc0(sizeof(MapCacheRev
));
339 reventry
->vaddr_req
= mapcache
->last_entry
->vaddr_base
+ address_offset
;
340 reventry
->paddr_index
= mapcache
->last_entry
->paddr_index
;
341 reventry
->size
= entry
->size
;
342 QTAILQ_INSERT_HEAD(&mapcache
->locked_entries
, reventry
, next
);
345 trace_xen_map_cache_return(mapcache
->last_entry
->vaddr_base
+ address_offset
);
346 return mapcache
->last_entry
->vaddr_base
+ address_offset
;
349 uint8_t *xen_map_cache(hwaddr phys_addr
, hwaddr size
,
350 uint8_t lock
, bool dma
)
355 p
= xen_map_cache_unlocked(phys_addr
, size
, lock
, dma
);
360 ram_addr_t
xen_ram_addr_from_mapcache(void *ptr
)
362 MapCacheEntry
*entry
= NULL
;
363 MapCacheRev
*reventry
;
370 QTAILQ_FOREACH(reventry
, &mapcache
->locked_entries
, next
) {
371 if (reventry
->vaddr_req
== ptr
) {
372 paddr_index
= reventry
->paddr_index
;
373 size
= reventry
->size
;
379 fprintf(stderr
, "%s, could not find %p\n", __func__
, ptr
);
380 QTAILQ_FOREACH(reventry
, &mapcache
->locked_entries
, next
) {
381 DPRINTF(" "TARGET_FMT_plx
" -> %p is present\n", reventry
->paddr_index
,
382 reventry
->vaddr_req
);
388 entry
= &mapcache
->entry
[paddr_index
% mapcache
->nr_buckets
];
389 while (entry
&& (entry
->paddr_index
!= paddr_index
|| entry
->size
!= size
)) {
393 DPRINTF("Trying to find address %p that is not in the mapcache!\n", ptr
);
396 raddr
= (reventry
->paddr_index
<< MCACHE_BUCKET_SHIFT
) +
397 ((unsigned long) ptr
- (unsigned long) entry
->vaddr_base
);
403 static void xen_invalidate_map_cache_entry_unlocked(uint8_t *buffer
)
405 MapCacheEntry
*entry
= NULL
, *pentry
= NULL
;
406 MapCacheRev
*reventry
;
411 QTAILQ_FOREACH(reventry
, &mapcache
->locked_entries
, next
) {
412 if (reventry
->vaddr_req
== buffer
) {
413 paddr_index
= reventry
->paddr_index
;
414 size
= reventry
->size
;
420 DPRINTF("%s, could not find %p\n", __func__
, buffer
);
421 QTAILQ_FOREACH(reventry
, &mapcache
->locked_entries
, next
) {
422 DPRINTF(" "TARGET_FMT_plx
" -> %p is present\n", reventry
->paddr_index
, reventry
->vaddr_req
);
426 QTAILQ_REMOVE(&mapcache
->locked_entries
, reventry
, next
);
429 if (mapcache
->last_entry
!= NULL
&&
430 mapcache
->last_entry
->paddr_index
== paddr_index
) {
431 mapcache
->last_entry
= NULL
;
434 entry
= &mapcache
->entry
[paddr_index
% mapcache
->nr_buckets
];
435 while (entry
&& (entry
->paddr_index
!= paddr_index
|| entry
->size
!= size
)) {
440 DPRINTF("Trying to unmap address %p that is not in the mapcache!\n", buffer
);
444 if (entry
->lock
> 0 || pentry
== NULL
) {
448 pentry
->next
= entry
->next
;
449 ram_block_notify_remove(entry
->vaddr_base
, entry
->size
);
450 if (munmap(entry
->vaddr_base
, entry
->size
) != 0) {
451 perror("unmap fails");
454 g_free(entry
->valid_mapping
);
458 void xen_invalidate_map_cache_entry(uint8_t *buffer
)
461 xen_invalidate_map_cache_entry_unlocked(buffer
);
465 void xen_invalidate_map_cache(void)
468 MapCacheRev
*reventry
;
470 /* Flush pending AIO before destroying the mapcache */
475 QTAILQ_FOREACH(reventry
, &mapcache
->locked_entries
, next
) {
476 if (!reventry
->dma
) {
479 fprintf(stderr
, "Locked DMA mapping while invalidating mapcache!"
480 " "TARGET_FMT_plx
" -> %p is present\n",
481 reventry
->paddr_index
, reventry
->vaddr_req
);
484 for (i
= 0; i
< mapcache
->nr_buckets
; i
++) {
485 MapCacheEntry
*entry
= &mapcache
->entry
[i
];
487 if (entry
->vaddr_base
== NULL
) {
490 if (entry
->lock
> 0) {
494 if (munmap(entry
->vaddr_base
, entry
->size
) != 0) {
495 perror("unmap fails");
499 entry
->paddr_index
= 0;
500 entry
->vaddr_base
= NULL
;
502 g_free(entry
->valid_mapping
);
503 entry
->valid_mapping
= NULL
;
506 mapcache
->last_entry
= NULL
;
511 static uint8_t *xen_replace_cache_entry_unlocked(hwaddr old_phys_addr
,
512 hwaddr new_phys_addr
,
515 MapCacheEntry
*entry
;
516 hwaddr address_index
, address_offset
;
517 hwaddr test_bit_size
, cache_size
= size
;
519 address_index
= old_phys_addr
>> MCACHE_BUCKET_SHIFT
;
520 address_offset
= old_phys_addr
& (MCACHE_BUCKET_SIZE
- 1);
523 /* test_bit_size is always a multiple of XC_PAGE_SIZE */
524 test_bit_size
= size
+ (old_phys_addr
& (XC_PAGE_SIZE
- 1));
525 if (test_bit_size
% XC_PAGE_SIZE
) {
526 test_bit_size
+= XC_PAGE_SIZE
- (test_bit_size
% XC_PAGE_SIZE
);
528 cache_size
= size
+ address_offset
;
529 if (cache_size
% MCACHE_BUCKET_SIZE
) {
530 cache_size
+= MCACHE_BUCKET_SIZE
- (cache_size
% MCACHE_BUCKET_SIZE
);
533 entry
= &mapcache
->entry
[address_index
% mapcache
->nr_buckets
];
534 while (entry
&& !(entry
->paddr_index
== address_index
&&
535 entry
->size
== cache_size
)) {
539 DPRINTF("Trying to update an entry for "TARGET_FMT_plx \
540 "that is not in the mapcache!\n", old_phys_addr
);
544 address_index
= new_phys_addr
>> MCACHE_BUCKET_SHIFT
;
545 address_offset
= new_phys_addr
& (MCACHE_BUCKET_SIZE
- 1);
547 fprintf(stderr
, "Replacing a dummy mapcache entry for "TARGET_FMT_plx \
548 " with "TARGET_FMT_plx
"\n", old_phys_addr
, new_phys_addr
);
550 xen_remap_bucket(entry
, entry
->vaddr_base
,
551 cache_size
, address_index
, false);
552 if (!test_bits(address_offset
>> XC_PAGE_SHIFT
,
553 test_bit_size
>> XC_PAGE_SHIFT
,
554 entry
->valid_mapping
)) {
555 DPRINTF("Unable to update a mapcache entry for "TARGET_FMT_plx
"!\n",
560 return entry
->vaddr_base
+ address_offset
;
563 uint8_t *xen_replace_cache_entry(hwaddr old_phys_addr
,
564 hwaddr new_phys_addr
,
570 p
= xen_replace_cache_entry_unlocked(old_phys_addr
, new_phys_addr
, size
);