2 * Copyright (C) 2011 Citrix Ltd.
4 * This work is licensed under the terms of the GNU GPL, version 2. See
5 * the COPYING file in the top-level directory.
7 * Contributions after 2012-01-13 are licensed under the terms of the
8 * GNU GPL, version 2 or (at your option) any later version.
11 #include "qemu/osdep.h"
12 #include "qemu/units.h"
13 #include "qemu/error-report.h"
15 #include <sys/resource.h>
17 #include "hw/xen/xen-legacy-backend.h"
18 #include "qemu/bitmap.h"
20 #include "sysemu/runstate.h"
21 #include "sysemu/xen-mapcache.h"
25 //#define MAPCACHE_DEBUG
28 # define DPRINTF(fmt, ...) do { \
29 fprintf(stderr, "xen_mapcache: " fmt, ## __VA_ARGS__); \
32 # define DPRINTF(fmt, ...) do { } while (0)
35 #if HOST_LONG_BITS == 32
36 # define MCACHE_BUCKET_SHIFT 16
37 # define MCACHE_MAX_SIZE (1UL<<31) /* 2GB Cap */
39 # define MCACHE_BUCKET_SHIFT 20
40 # define MCACHE_MAX_SIZE (1UL<<35) /* 32GB Cap */
42 #define MCACHE_BUCKET_SIZE (1UL << MCACHE_BUCKET_SHIFT)
44 /* This is the size of the virtual address space reserve to QEMU that will not
46 * From empirical tests I observed that qemu use 75MB more than the
49 #define NON_MCACHE_MEMORY_SIZE (80 * MiB)
51 typedef struct MapCacheEntry
{
54 unsigned long *valid_mapping
;
56 #define XEN_MAPCACHE_ENTRY_DUMMY (1 << 0)
59 struct MapCacheEntry
*next
;
62 typedef struct MapCacheRev
{
66 QTAILQ_ENTRY(MapCacheRev
) next
;
70 typedef struct MapCache
{
72 unsigned long nr_buckets
;
73 QTAILQ_HEAD(, MapCacheRev
) locked_entries
;
75 /* For most cases (>99.9%), the page address is the same. */
76 MapCacheEntry
*last_entry
;
77 unsigned long max_mcache_size
;
78 unsigned int mcache_bucket_shift
;
80 phys_offset_to_gaddr_t phys_offset_to_gaddr
;
85 static MapCache
*mapcache
;
87 static inline void mapcache_lock(void)
89 qemu_mutex_lock(&mapcache
->lock
);
92 static inline void mapcache_unlock(void)
94 qemu_mutex_unlock(&mapcache
->lock
);
97 static inline int test_bits(int nr
, int size
, const unsigned long *addr
)
99 unsigned long res
= find_next_zero_bit(addr
, size
+ nr
, nr
);
100 if (res
>= nr
+ size
)
106 void xen_map_cache_init(phys_offset_to_gaddr_t f
, void *opaque
)
109 struct rlimit rlimit_as
;
111 mapcache
= g_malloc0(sizeof (MapCache
));
113 mapcache
->phys_offset_to_gaddr
= f
;
114 mapcache
->opaque
= opaque
;
115 qemu_mutex_init(&mapcache
->lock
);
117 QTAILQ_INIT(&mapcache
->locked_entries
);
119 if (geteuid() == 0) {
120 rlimit_as
.rlim_cur
= RLIM_INFINITY
;
121 rlimit_as
.rlim_max
= RLIM_INFINITY
;
122 mapcache
->max_mcache_size
= MCACHE_MAX_SIZE
;
124 getrlimit(RLIMIT_AS
, &rlimit_as
);
125 rlimit_as
.rlim_cur
= rlimit_as
.rlim_max
;
127 if (rlimit_as
.rlim_max
!= RLIM_INFINITY
) {
128 warn_report("QEMU's maximum size of virtual"
129 " memory is not infinity");
131 if (rlimit_as
.rlim_max
< MCACHE_MAX_SIZE
+ NON_MCACHE_MEMORY_SIZE
) {
132 mapcache
->max_mcache_size
= rlimit_as
.rlim_max
-
133 NON_MCACHE_MEMORY_SIZE
;
135 mapcache
->max_mcache_size
= MCACHE_MAX_SIZE
;
139 setrlimit(RLIMIT_AS
, &rlimit_as
);
141 mapcache
->nr_buckets
=
142 (((mapcache
->max_mcache_size
>> XC_PAGE_SHIFT
) +
143 (1UL << (MCACHE_BUCKET_SHIFT
- XC_PAGE_SHIFT
)) - 1) >>
144 (MCACHE_BUCKET_SHIFT
- XC_PAGE_SHIFT
));
146 size
= mapcache
->nr_buckets
* sizeof (MapCacheEntry
);
147 size
= (size
+ XC_PAGE_SIZE
- 1) & ~(XC_PAGE_SIZE
- 1);
148 DPRINTF("%s, nr_buckets = %lx size %lu\n", __func__
,
149 mapcache
->nr_buckets
, size
);
150 mapcache
->entry
= g_malloc0(size
);
153 static void xen_remap_bucket(MapCacheEntry
*entry
,
156 hwaddr address_index
,
163 hwaddr nb_pfn
= size
>> XC_PAGE_SHIFT
;
165 trace_xen_remap_bucket(address_index
);
167 pfns
= g_malloc0(nb_pfn
* sizeof (xen_pfn_t
));
168 err
= g_malloc0(nb_pfn
* sizeof (int));
170 if (entry
->vaddr_base
!= NULL
) {
171 if (!(entry
->flags
& XEN_MAPCACHE_ENTRY_DUMMY
)) {
172 ram_block_notify_remove(entry
->vaddr_base
, entry
->size
);
174 if (munmap(entry
->vaddr_base
, entry
->size
) != 0) {
175 perror("unmap fails");
179 g_free(entry
->valid_mapping
);
180 entry
->valid_mapping
= NULL
;
182 for (i
= 0; i
< nb_pfn
; i
++) {
183 pfns
[i
] = (address_index
<< (MCACHE_BUCKET_SHIFT
-XC_PAGE_SHIFT
)) + i
;
187 * If the caller has requested the mapping at a specific address use
188 * MAP_FIXED to make sure it's honored.
191 vaddr_base
= xenforeignmemory_map2(xen_fmem
, xen_domid
, vaddr
,
192 PROT_READ
| PROT_WRITE
,
193 vaddr
? MAP_FIXED
: 0,
195 if (vaddr_base
== NULL
) {
196 perror("xenforeignmemory_map2");
201 * We create dummy mappings where we are unable to create a foreign
202 * mapping immediately due to certain circumstances (i.e. on resume now)
204 vaddr_base
= mmap(vaddr
, size
, PROT_READ
| PROT_WRITE
,
205 MAP_ANON
| MAP_SHARED
| (vaddr
? MAP_FIXED
: 0),
207 if (vaddr_base
== MAP_FAILED
) {
213 if (!(entry
->flags
& XEN_MAPCACHE_ENTRY_DUMMY
)) {
214 ram_block_notify_add(vaddr_base
, size
);
217 entry
->vaddr_base
= vaddr_base
;
218 entry
->paddr_index
= address_index
;
220 entry
->valid_mapping
= (unsigned long *) g_malloc0(sizeof(unsigned long) *
221 BITS_TO_LONGS(size
>> XC_PAGE_SHIFT
));
224 entry
->flags
|= XEN_MAPCACHE_ENTRY_DUMMY
;
226 entry
->flags
&= ~(XEN_MAPCACHE_ENTRY_DUMMY
);
229 bitmap_zero(entry
->valid_mapping
, nb_pfn
);
230 for (i
= 0; i
< nb_pfn
; i
++) {
232 bitmap_set(entry
->valid_mapping
, i
, 1);
240 static uint8_t *xen_map_cache_unlocked(hwaddr phys_addr
, hwaddr size
,
241 uint8_t lock
, bool dma
)
243 MapCacheEntry
*entry
, *pentry
= NULL
,
244 *free_entry
= NULL
, *free_pentry
= NULL
;
245 hwaddr address_index
;
246 hwaddr address_offset
;
247 hwaddr cache_size
= size
;
248 hwaddr test_bit_size
;
249 bool translated G_GNUC_UNUSED
= false;
253 address_index
= phys_addr
>> MCACHE_BUCKET_SHIFT
;
254 address_offset
= phys_addr
& (MCACHE_BUCKET_SIZE
- 1);
256 trace_xen_map_cache(phys_addr
);
258 /* test_bit_size is always a multiple of XC_PAGE_SIZE */
260 test_bit_size
= size
+ (phys_addr
& (XC_PAGE_SIZE
- 1));
262 if (test_bit_size
% XC_PAGE_SIZE
) {
263 test_bit_size
+= XC_PAGE_SIZE
- (test_bit_size
% XC_PAGE_SIZE
);
266 test_bit_size
= XC_PAGE_SIZE
;
269 if (mapcache
->last_entry
!= NULL
&&
270 mapcache
->last_entry
->paddr_index
== address_index
&&
272 test_bits(address_offset
>> XC_PAGE_SHIFT
,
273 test_bit_size
>> XC_PAGE_SHIFT
,
274 mapcache
->last_entry
->valid_mapping
)) {
275 trace_xen_map_cache_return(mapcache
->last_entry
->vaddr_base
+ address_offset
);
276 return mapcache
->last_entry
->vaddr_base
+ address_offset
;
279 /* size is always a multiple of MCACHE_BUCKET_SIZE */
281 cache_size
= size
+ address_offset
;
282 if (cache_size
% MCACHE_BUCKET_SIZE
) {
283 cache_size
+= MCACHE_BUCKET_SIZE
- (cache_size
% MCACHE_BUCKET_SIZE
);
286 cache_size
= MCACHE_BUCKET_SIZE
;
289 entry
= &mapcache
->entry
[address_index
% mapcache
->nr_buckets
];
291 while (entry
&& (lock
|| entry
->lock
) && entry
->vaddr_base
&&
292 (entry
->paddr_index
!= address_index
|| entry
->size
!= cache_size
||
293 !test_bits(address_offset
>> XC_PAGE_SHIFT
,
294 test_bit_size
>> XC_PAGE_SHIFT
,
295 entry
->valid_mapping
))) {
296 if (!free_entry
&& !entry
->lock
) {
298 free_pentry
= pentry
;
303 if (!entry
&& free_entry
) {
305 pentry
= free_pentry
;
308 entry
= g_malloc0(sizeof (MapCacheEntry
));
309 pentry
->next
= entry
;
310 xen_remap_bucket(entry
, NULL
, cache_size
, address_index
, dummy
);
311 } else if (!entry
->lock
) {
312 if (!entry
->vaddr_base
|| entry
->paddr_index
!= address_index
||
313 entry
->size
!= cache_size
||
314 !test_bits(address_offset
>> XC_PAGE_SHIFT
,
315 test_bit_size
>> XC_PAGE_SHIFT
,
316 entry
->valid_mapping
)) {
317 xen_remap_bucket(entry
, NULL
, cache_size
, address_index
, dummy
);
321 if(!test_bits(address_offset
>> XC_PAGE_SHIFT
,
322 test_bit_size
>> XC_PAGE_SHIFT
,
323 entry
->valid_mapping
)) {
324 mapcache
->last_entry
= NULL
;
325 #ifdef XEN_COMPAT_PHYSMAP
326 if (!translated
&& mapcache
->phys_offset_to_gaddr
) {
327 phys_addr
= mapcache
->phys_offset_to_gaddr(phys_addr
, size
);
332 if (!dummy
&& runstate_check(RUN_STATE_INMIGRATE
)) {
336 trace_xen_map_cache_return(NULL
);
340 mapcache
->last_entry
= entry
;
342 MapCacheRev
*reventry
= g_malloc0(sizeof(MapCacheRev
));
345 reventry
->vaddr_req
= mapcache
->last_entry
->vaddr_base
+ address_offset
;
346 reventry
->paddr_index
= mapcache
->last_entry
->paddr_index
;
347 reventry
->size
= entry
->size
;
348 QTAILQ_INSERT_HEAD(&mapcache
->locked_entries
, reventry
, next
);
351 trace_xen_map_cache_return(mapcache
->last_entry
->vaddr_base
+ address_offset
);
352 return mapcache
->last_entry
->vaddr_base
+ address_offset
;
355 uint8_t *xen_map_cache(hwaddr phys_addr
, hwaddr size
,
356 uint8_t lock
, bool dma
)
361 p
= xen_map_cache_unlocked(phys_addr
, size
, lock
, dma
);
366 ram_addr_t
xen_ram_addr_from_mapcache(void *ptr
)
368 MapCacheEntry
*entry
= NULL
;
369 MapCacheRev
*reventry
;
376 QTAILQ_FOREACH(reventry
, &mapcache
->locked_entries
, next
) {
377 if (reventry
->vaddr_req
== ptr
) {
378 paddr_index
= reventry
->paddr_index
;
379 size
= reventry
->size
;
385 fprintf(stderr
, "%s, could not find %p\n", __func__
, ptr
);
386 QTAILQ_FOREACH(reventry
, &mapcache
->locked_entries
, next
) {
387 DPRINTF(" "TARGET_FMT_plx
" -> %p is present\n", reventry
->paddr_index
,
388 reventry
->vaddr_req
);
394 entry
= &mapcache
->entry
[paddr_index
% mapcache
->nr_buckets
];
395 while (entry
&& (entry
->paddr_index
!= paddr_index
|| entry
->size
!= size
)) {
399 DPRINTF("Trying to find address %p that is not in the mapcache!\n", ptr
);
402 raddr
= (reventry
->paddr_index
<< MCACHE_BUCKET_SHIFT
) +
403 ((unsigned long) ptr
- (unsigned long) entry
->vaddr_base
);
409 static void xen_invalidate_map_cache_entry_unlocked(uint8_t *buffer
)
411 MapCacheEntry
*entry
= NULL
, *pentry
= NULL
;
412 MapCacheRev
*reventry
;
417 QTAILQ_FOREACH(reventry
, &mapcache
->locked_entries
, next
) {
418 if (reventry
->vaddr_req
== buffer
) {
419 paddr_index
= reventry
->paddr_index
;
420 size
= reventry
->size
;
426 DPRINTF("%s, could not find %p\n", __func__
, buffer
);
427 QTAILQ_FOREACH(reventry
, &mapcache
->locked_entries
, next
) {
428 DPRINTF(" "TARGET_FMT_plx
" -> %p is present\n", reventry
->paddr_index
, reventry
->vaddr_req
);
432 QTAILQ_REMOVE(&mapcache
->locked_entries
, reventry
, next
);
435 if (mapcache
->last_entry
!= NULL
&&
436 mapcache
->last_entry
->paddr_index
== paddr_index
) {
437 mapcache
->last_entry
= NULL
;
440 entry
= &mapcache
->entry
[paddr_index
% mapcache
->nr_buckets
];
441 while (entry
&& (entry
->paddr_index
!= paddr_index
|| entry
->size
!= size
)) {
446 DPRINTF("Trying to unmap address %p that is not in the mapcache!\n", buffer
);
450 if (entry
->lock
> 0 || pentry
== NULL
) {
454 pentry
->next
= entry
->next
;
455 ram_block_notify_remove(entry
->vaddr_base
, entry
->size
);
456 if (munmap(entry
->vaddr_base
, entry
->size
) != 0) {
457 perror("unmap fails");
460 g_free(entry
->valid_mapping
);
464 void xen_invalidate_map_cache_entry(uint8_t *buffer
)
467 xen_invalidate_map_cache_entry_unlocked(buffer
);
471 void xen_invalidate_map_cache(void)
474 MapCacheRev
*reventry
;
476 /* Flush pending AIO before destroying the mapcache */
481 QTAILQ_FOREACH(reventry
, &mapcache
->locked_entries
, next
) {
482 if (!reventry
->dma
) {
485 fprintf(stderr
, "Locked DMA mapping while invalidating mapcache!"
486 " "TARGET_FMT_plx
" -> %p is present\n",
487 reventry
->paddr_index
, reventry
->vaddr_req
);
490 for (i
= 0; i
< mapcache
->nr_buckets
; i
++) {
491 MapCacheEntry
*entry
= &mapcache
->entry
[i
];
493 if (entry
->vaddr_base
== NULL
) {
496 if (entry
->lock
> 0) {
500 if (munmap(entry
->vaddr_base
, entry
->size
) != 0) {
501 perror("unmap fails");
505 entry
->paddr_index
= 0;
506 entry
->vaddr_base
= NULL
;
508 g_free(entry
->valid_mapping
);
509 entry
->valid_mapping
= NULL
;
512 mapcache
->last_entry
= NULL
;
517 static uint8_t *xen_replace_cache_entry_unlocked(hwaddr old_phys_addr
,
518 hwaddr new_phys_addr
,
521 MapCacheEntry
*entry
;
522 hwaddr address_index
, address_offset
;
523 hwaddr test_bit_size
, cache_size
= size
;
525 address_index
= old_phys_addr
>> MCACHE_BUCKET_SHIFT
;
526 address_offset
= old_phys_addr
& (MCACHE_BUCKET_SIZE
- 1);
529 /* test_bit_size is always a multiple of XC_PAGE_SIZE */
530 test_bit_size
= size
+ (old_phys_addr
& (XC_PAGE_SIZE
- 1));
531 if (test_bit_size
% XC_PAGE_SIZE
) {
532 test_bit_size
+= XC_PAGE_SIZE
- (test_bit_size
% XC_PAGE_SIZE
);
534 cache_size
= size
+ address_offset
;
535 if (cache_size
% MCACHE_BUCKET_SIZE
) {
536 cache_size
+= MCACHE_BUCKET_SIZE
- (cache_size
% MCACHE_BUCKET_SIZE
);
539 entry
= &mapcache
->entry
[address_index
% mapcache
->nr_buckets
];
540 while (entry
&& !(entry
->paddr_index
== address_index
&&
541 entry
->size
== cache_size
)) {
545 DPRINTF("Trying to update an entry for "TARGET_FMT_plx \
546 "that is not in the mapcache!\n", old_phys_addr
);
550 address_index
= new_phys_addr
>> MCACHE_BUCKET_SHIFT
;
551 address_offset
= new_phys_addr
& (MCACHE_BUCKET_SIZE
- 1);
553 fprintf(stderr
, "Replacing a dummy mapcache entry for "TARGET_FMT_plx \
554 " with "TARGET_FMT_plx
"\n", old_phys_addr
, new_phys_addr
);
556 xen_remap_bucket(entry
, entry
->vaddr_base
,
557 cache_size
, address_index
, false);
558 if (!test_bits(address_offset
>> XC_PAGE_SHIFT
,
559 test_bit_size
>> XC_PAGE_SHIFT
,
560 entry
->valid_mapping
)) {
561 DPRINTF("Unable to update a mapcache entry for "TARGET_FMT_plx
"!\n",
566 return entry
->vaddr_base
+ address_offset
;
569 uint8_t *xen_replace_cache_entry(hwaddr old_phys_addr
,
570 hwaddr new_phys_addr
,
576 p
= xen_replace_cache_entry_unlocked(old_phys_addr
, new_phys_addr
, size
);