2 * mmap support for qemu
4 * Copyright (c) 2003 Fabrice Bellard
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, see <http://www.gnu.org/licenses/>.
19 #include "qemu/osdep.h"
26 static pthread_mutex_t mmap_mutex
= PTHREAD_MUTEX_INITIALIZER
;
27 static __thread
int mmap_lock_count
;
31 if (mmap_lock_count
++ == 0) {
32 pthread_mutex_lock(&mmap_mutex
);
36 void mmap_unlock(void)
38 if (--mmap_lock_count
== 0) {
39 pthread_mutex_unlock(&mmap_mutex
);
43 bool have_mmap_lock(void)
45 return mmap_lock_count
> 0 ? true : false;
48 /* Grab lock to make sure things are in a consistent state after fork(). */
49 void mmap_fork_start(void)
53 pthread_mutex_lock(&mmap_mutex
);
56 void mmap_fork_end(int child
)
59 pthread_mutex_init(&mmap_mutex
, NULL
);
61 pthread_mutex_unlock(&mmap_mutex
);
64 /* NOTE: all the constants are the HOST ones, but addresses are target. */
65 int target_mprotect(abi_ulong start
, abi_ulong len
, int prot
)
67 abi_ulong end
, host_start
, host_end
, addr
;
70 trace_target_mprotect(start
, len
, prot
);
72 if ((start
& ~TARGET_PAGE_MASK
) != 0)
73 return -TARGET_EINVAL
;
74 len
= TARGET_PAGE_ALIGN(len
);
76 if (!guest_range_valid(start
, len
)) {
77 return -TARGET_ENOMEM
;
79 prot
&= PROT_READ
| PROT_WRITE
| PROT_EXEC
;
84 host_start
= start
& qemu_host_page_mask
;
85 host_end
= HOST_PAGE_ALIGN(end
);
86 if (start
> host_start
) {
87 /* handle host page containing start */
89 for(addr
= host_start
; addr
< start
; addr
+= TARGET_PAGE_SIZE
) {
90 prot1
|= page_get_flags(addr
);
92 if (host_end
== host_start
+ qemu_host_page_size
) {
93 for(addr
= end
; addr
< host_end
; addr
+= TARGET_PAGE_SIZE
) {
94 prot1
|= page_get_flags(addr
);
98 ret
= mprotect(g2h(host_start
), qemu_host_page_size
, prot1
& PAGE_BITS
);
101 host_start
+= qemu_host_page_size
;
103 if (end
< host_end
) {
105 for(addr
= end
; addr
< host_end
; addr
+= TARGET_PAGE_SIZE
) {
106 prot1
|= page_get_flags(addr
);
108 ret
= mprotect(g2h(host_end
- qemu_host_page_size
), qemu_host_page_size
,
112 host_end
-= qemu_host_page_size
;
115 /* handle the pages in the middle */
116 if (host_start
< host_end
) {
117 ret
= mprotect(g2h(host_start
), host_end
- host_start
, prot
);
121 page_set_flags(start
, start
+ len
, prot
| PAGE_VALID
);
129 /* map an incomplete host page */
130 static int mmap_frag(abi_ulong real_start
,
131 abi_ulong start
, abi_ulong end
,
132 int prot
, int flags
, int fd
, abi_ulong offset
)
134 abi_ulong real_end
, addr
;
138 real_end
= real_start
+ qemu_host_page_size
;
139 host_start
= g2h(real_start
);
141 /* get the protection of the target pages outside the mapping */
143 for(addr
= real_start
; addr
< real_end
; addr
++) {
144 if (addr
< start
|| addr
>= end
)
145 prot1
|= page_get_flags(addr
);
149 /* no page was there, so we allocate one */
150 void *p
= mmap(host_start
, qemu_host_page_size
, prot
,
151 flags
| MAP_ANONYMOUS
, -1, 0);
158 prot_new
= prot
| prot1
;
159 if (!(flags
& MAP_ANONYMOUS
)) {
160 /* msync() won't work here, so we return an error if write is
161 possible while it is a shared mapping */
162 if ((flags
& MAP_TYPE
) == MAP_SHARED
&&
166 /* adjust protection to be able to read */
167 if (!(prot1
& PROT_WRITE
))
168 mprotect(host_start
, qemu_host_page_size
, prot1
| PROT_WRITE
);
170 /* read the corresponding file data */
171 if (pread(fd
, g2h(start
), end
- start
, offset
) == -1)
174 /* put final protection */
175 if (prot_new
!= (prot1
| PROT_WRITE
))
176 mprotect(host_start
, qemu_host_page_size
, prot_new
);
178 if (prot_new
!= prot1
) {
179 mprotect(host_start
, qemu_host_page_size
, prot_new
);
181 if (prot_new
& PROT_WRITE
) {
182 memset(g2h(start
), 0, end
- start
);
188 #if HOST_LONG_BITS == 64 && TARGET_ABI_BITS == 64
189 # define TASK_UNMAPPED_BASE (1ul << 38)
191 # define TASK_UNMAPPED_BASE 0x40000000
193 abi_ulong mmap_next_start
= TASK_UNMAPPED_BASE
;
195 unsigned long last_brk
;
197 /* Subroutine of mmap_find_vma, used when we have pre-allocated a chunk
198 of guest address space. */
199 static abi_ulong
mmap_find_vma_reserved(abi_ulong start
, abi_ulong size
,
202 abi_ulong addr
, end_addr
, incr
= qemu_host_page_size
;
206 if (size
> reserved_va
) {
207 return (abi_ulong
)-1;
210 /* Note that start and size have already been aligned by mmap_find_vma. */
212 end_addr
= start
+ size
;
213 if (start
> reserved_va
- size
) {
214 /* Start at the top of the address space. */
215 end_addr
= ((reserved_va
- size
) & -align
) + size
;
219 /* Search downward from END_ADDR, checking to see if a page is in use. */
223 if (addr
> end_addr
) {
225 /* Failure. The entire address space has been searched. */
226 return (abi_ulong
)-1;
228 /* Re-start at the top of the address space. */
229 addr
= end_addr
= ((reserved_va
- size
) & -align
) + size
;
232 prot
= page_get_flags(addr
);
234 /* Page in use. Restart below this page. */
235 addr
= end_addr
= ((addr
- size
) & -align
) + size
;
236 } else if (addr
&& addr
+ size
== end_addr
) {
237 /* Success! All pages between ADDR and END_ADDR are free. */
238 if (start
== mmap_next_start
) {
239 mmap_next_start
= addr
;
248 * Find and reserve a free memory area of size 'size'. The search
250 * It must be called with mmap_lock() held.
251 * Return -1 if error.
253 abi_ulong
mmap_find_vma(abi_ulong start
, abi_ulong size
, abi_ulong align
)
259 align
= MAX(align
, qemu_host_page_size
);
261 /* If 'start' == 0, then a default start address is used. */
263 start
= mmap_next_start
;
265 start
&= qemu_host_page_mask
;
267 start
= ROUND_UP(start
, align
);
269 size
= HOST_PAGE_ALIGN(size
);
272 return mmap_find_vma_reserved(start
, size
, align
);
276 wrapped
= repeat
= 0;
279 for (;; prev
= ptr
) {
281 * Reserve needed memory area to avoid a race.
282 * It should be discarded using:
283 * - mmap() with MAP_FIXED flag
284 * - mremap() with MREMAP_FIXED flag
285 * - shmat() with SHM_REMAP flag
287 ptr
= mmap(g2h(addr
), size
, PROT_NONE
,
288 MAP_ANONYMOUS
|MAP_PRIVATE
|MAP_NORESERVE
, -1, 0);
290 /* ENOMEM, if host address space has no memory */
291 if (ptr
== MAP_FAILED
) {
292 return (abi_ulong
)-1;
295 /* Count the number of sequential returns of the same address.
296 This is used to modify the search algorithm below. */
297 repeat
= (ptr
== prev
? repeat
+ 1 : 0);
299 if (h2g_valid(ptr
+ size
- 1)) {
302 if ((addr
& (align
- 1)) == 0) {
304 if (start
== mmap_next_start
&& addr
>= TASK_UNMAPPED_BASE
) {
305 mmap_next_start
= addr
+ size
;
310 /* The address is not properly aligned for the target. */
313 /* Assume the result that the kernel gave us is the
314 first with enough free space, so start again at the
315 next higher target page. */
316 addr
= ROUND_UP(addr
, align
);
319 /* Sometimes the kernel decides to perform the allocation
320 at the top end of memory instead. */
324 /* Start over at low memory. */
328 /* Fail. This unaligned block must the last. */
333 /* Since the result the kernel gave didn't fit, start
334 again at low memory. If any repetition, fail. */
335 addr
= (repeat
? -1 : 0);
338 /* Unmap and try again. */
341 /* ENOMEM if we checked the whole of the target address space. */
342 if (addr
== (abi_ulong
)-1) {
343 return (abi_ulong
)-1;
344 } else if (addr
== 0) {
346 return (abi_ulong
)-1;
349 /* Don't actually use 0 when wrapping, instead indicate
350 that we'd truly like an allocation in low memory. */
351 addr
= (mmap_min_addr
> TARGET_PAGE_SIZE
352 ? TARGET_PAGE_ALIGN(mmap_min_addr
)
354 } else if (wrapped
&& addr
>= start
) {
355 return (abi_ulong
)-1;
360 /* NOTE: all the constants are the HOST ones */
361 abi_long
target_mmap(abi_ulong start
, abi_ulong len
, int prot
,
362 int flags
, int fd
, abi_ulong offset
)
364 abi_ulong ret
, end
, real_start
, real_end
, retaddr
, host_offset
, host_len
;
367 trace_target_mmap(start
, len
, prot
, flags
, fd
, offset
);
374 /* Also check for overflows... */
375 len
= TARGET_PAGE_ALIGN(len
);
381 if (offset
& ~TARGET_PAGE_MASK
) {
386 real_start
= start
& qemu_host_page_mask
;
387 host_offset
= offset
& qemu_host_page_mask
;
389 /* If the user is asking for the kernel to find a location, do that
390 before we truncate the length for mapping files below. */
391 if (!(flags
& MAP_FIXED
)) {
392 host_len
= len
+ offset
- host_offset
;
393 host_len
= HOST_PAGE_ALIGN(host_len
);
394 start
= mmap_find_vma(real_start
, host_len
, TARGET_PAGE_SIZE
);
395 if (start
== (abi_ulong
)-1) {
401 /* When mapping files into a memory area larger than the file, accesses
402 to pages beyond the file size will cause a SIGBUS.
404 For example, if mmaping a file of 100 bytes on a host with 4K pages
405 emulating a target with 8K pages, the target expects to be able to
406 access the first 8K. But the host will trap us on any access beyond
409 When emulating a target with a larger page-size than the hosts, we
410 may need to truncate file maps at EOF and add extra anonymous pages
411 up to the targets page boundary. */
413 if ((qemu_real_host_page_size
< qemu_host_page_size
) &&
414 !(flags
& MAP_ANONYMOUS
)) {
417 if (fstat (fd
, &sb
) == -1)
420 /* Are we trying to create a map beyond EOF?. */
421 if (offset
+ len
> sb
.st_size
) {
422 /* If so, truncate the file map at eof aligned with
423 the hosts real pagesize. Additional anonymous maps
424 will be created beyond EOF. */
425 len
= REAL_HOST_PAGE_ALIGN(sb
.st_size
- offset
);
429 if (!(flags
& MAP_FIXED
)) {
430 unsigned long host_start
;
433 host_len
= len
+ offset
- host_offset
;
434 host_len
= HOST_PAGE_ALIGN(host_len
);
436 /* Note: we prefer to control the mapping address. It is
437 especially important if qemu_host_page_size >
438 qemu_real_host_page_size */
439 p
= mmap(g2h(start
), host_len
, prot
,
440 flags
| MAP_FIXED
| MAP_ANONYMOUS
, -1, 0);
443 /* update start so that it points to the file position at 'offset' */
444 host_start
= (unsigned long)p
;
445 if (!(flags
& MAP_ANONYMOUS
)) {
446 p
= mmap(g2h(start
), len
, prot
,
447 flags
| MAP_FIXED
, fd
, host_offset
);
448 if (p
== MAP_FAILED
) {
449 munmap(g2h(start
), host_len
);
452 host_start
+= offset
- host_offset
;
454 start
= h2g(host_start
);
456 if (start
& ~TARGET_PAGE_MASK
) {
461 real_end
= HOST_PAGE_ALIGN(end
);
464 * Test if requested memory area fits target address space
465 * It can fail only on 64-bit host with 32-bit target.
466 * On any other target/host host mmap() handles this error correctly.
468 if (!guest_range_valid(start
, len
)) {
473 /* worst case: we cannot map the file because the offset is not
474 aligned, so we read it */
475 if (!(flags
& MAP_ANONYMOUS
) &&
476 (offset
& ~qemu_host_page_mask
) != (start
& ~qemu_host_page_mask
)) {
477 /* msync() won't work here, so we return an error if write is
478 possible while it is a shared mapping */
479 if ((flags
& MAP_TYPE
) == MAP_SHARED
&&
480 (prot
& PROT_WRITE
)) {
484 retaddr
= target_mmap(start
, len
, prot
| PROT_WRITE
,
485 MAP_FIXED
| MAP_PRIVATE
| MAP_ANONYMOUS
,
489 if (pread(fd
, g2h(start
), len
, offset
) == -1)
491 if (!(prot
& PROT_WRITE
)) {
492 ret
= target_mprotect(start
, len
, prot
);
498 /* handle the start of the mapping */
499 if (start
> real_start
) {
500 if (real_end
== real_start
+ qemu_host_page_size
) {
501 /* one single host page */
502 ret
= mmap_frag(real_start
, start
, end
,
503 prot
, flags
, fd
, offset
);
508 ret
= mmap_frag(real_start
, start
, real_start
+ qemu_host_page_size
,
509 prot
, flags
, fd
, offset
);
512 real_start
+= qemu_host_page_size
;
514 /* handle the end of the mapping */
515 if (end
< real_end
) {
516 ret
= mmap_frag(real_end
- qemu_host_page_size
,
517 real_end
- qemu_host_page_size
, end
,
519 offset
+ real_end
- qemu_host_page_size
- start
);
522 real_end
-= qemu_host_page_size
;
525 /* map the middle (easier) */
526 if (real_start
< real_end
) {
528 unsigned long offset1
;
529 if (flags
& MAP_ANONYMOUS
)
532 offset1
= offset
+ real_start
- start
;
533 p
= mmap(g2h(real_start
), real_end
- real_start
,
534 prot
, flags
, fd
, offset1
);
540 page_set_flags(start
, start
+ len
, prot
| PAGE_VALID
);
542 trace_target_mmap_complete(start
);
543 if (qemu_loglevel_mask(CPU_LOG_PAGE
)) {
544 log_page_dump(__func__
);
546 tb_invalidate_phys_range(start
, start
+ len
);
554 static void mmap_reserve(abi_ulong start
, abi_ulong size
)
556 abi_ulong real_start
;
562 real_start
= start
& qemu_host_page_mask
;
563 real_end
= HOST_PAGE_ALIGN(start
+ size
);
565 if (start
> real_start
) {
566 /* handle host page containing start */
568 for (addr
= real_start
; addr
< start
; addr
+= TARGET_PAGE_SIZE
) {
569 prot
|= page_get_flags(addr
);
571 if (real_end
== real_start
+ qemu_host_page_size
) {
572 for (addr
= end
; addr
< real_end
; addr
+= TARGET_PAGE_SIZE
) {
573 prot
|= page_get_flags(addr
);
578 real_start
+= qemu_host_page_size
;
580 if (end
< real_end
) {
582 for (addr
= end
; addr
< real_end
; addr
+= TARGET_PAGE_SIZE
) {
583 prot
|= page_get_flags(addr
);
586 real_end
-= qemu_host_page_size
;
588 if (real_start
!= real_end
) {
589 mmap(g2h(real_start
), real_end
- real_start
, PROT_NONE
,
590 MAP_FIXED
| MAP_ANONYMOUS
| MAP_PRIVATE
| MAP_NORESERVE
,
595 int target_munmap(abi_ulong start
, abi_ulong len
)
597 abi_ulong end
, real_start
, real_end
, addr
;
601 printf("munmap: start=0x" TARGET_ABI_FMT_lx
" len=0x"
602 TARGET_ABI_FMT_lx
"\n",
605 if (start
& ~TARGET_PAGE_MASK
)
606 return -TARGET_EINVAL
;
607 len
= TARGET_PAGE_ALIGN(len
);
608 if (len
== 0 || !guest_range_valid(start
, len
)) {
609 return -TARGET_EINVAL
;
614 real_start
= start
& qemu_host_page_mask
;
615 real_end
= HOST_PAGE_ALIGN(end
);
617 if (start
> real_start
) {
618 /* handle host page containing start */
620 for(addr
= real_start
; addr
< start
; addr
+= TARGET_PAGE_SIZE
) {
621 prot
|= page_get_flags(addr
);
623 if (real_end
== real_start
+ qemu_host_page_size
) {
624 for(addr
= end
; addr
< real_end
; addr
+= TARGET_PAGE_SIZE
) {
625 prot
|= page_get_flags(addr
);
630 real_start
+= qemu_host_page_size
;
632 if (end
< real_end
) {
634 for(addr
= end
; addr
< real_end
; addr
+= TARGET_PAGE_SIZE
) {
635 prot
|= page_get_flags(addr
);
638 real_end
-= qemu_host_page_size
;
642 /* unmap what we can */
643 if (real_start
< real_end
) {
645 mmap_reserve(real_start
, real_end
- real_start
);
647 ret
= munmap(g2h(real_start
), real_end
- real_start
);
652 page_set_flags(start
, start
+ len
, 0);
653 tb_invalidate_phys_range(start
, start
+ len
);
659 abi_long
target_mremap(abi_ulong old_addr
, abi_ulong old_size
,
660 abi_ulong new_size
, unsigned long flags
,
666 if (!guest_range_valid(old_addr
, old_size
) ||
667 ((flags
& MREMAP_FIXED
) &&
668 !guest_range_valid(new_addr
, new_size
))) {
675 if (flags
& MREMAP_FIXED
) {
676 host_addr
= mremap(g2h(old_addr
), old_size
, new_size
,
677 flags
, g2h(new_addr
));
679 if (reserved_va
&& host_addr
!= MAP_FAILED
) {
680 /* If new and old addresses overlap then the above mremap will
681 already have failed with EINVAL. */
682 mmap_reserve(old_addr
, old_size
);
684 } else if (flags
& MREMAP_MAYMOVE
) {
685 abi_ulong mmap_start
;
687 mmap_start
= mmap_find_vma(0, new_size
, TARGET_PAGE_SIZE
);
689 if (mmap_start
== -1) {
691 host_addr
= MAP_FAILED
;
693 host_addr
= mremap(g2h(old_addr
), old_size
, new_size
,
694 flags
| MREMAP_FIXED
, g2h(mmap_start
));
696 mmap_reserve(old_addr
, old_size
);
701 if (reserved_va
&& old_size
< new_size
) {
703 for (addr
= old_addr
+ old_size
;
704 addr
< old_addr
+ new_size
;
706 prot
|= page_get_flags(addr
);
710 host_addr
= mremap(g2h(old_addr
), old_size
, new_size
, flags
);
711 if (host_addr
!= MAP_FAILED
&& reserved_va
&& old_size
> new_size
) {
712 mmap_reserve(old_addr
+ old_size
, new_size
- old_size
);
716 host_addr
= MAP_FAILED
;
718 /* Check if address fits target address space */
719 if ((unsigned long)host_addr
+ new_size
> (abi_ulong
)-1) {
720 /* Revert mremap() changes */
721 host_addr
= mremap(g2h(old_addr
), new_size
, old_size
, flags
);
723 host_addr
= MAP_FAILED
;
727 if (host_addr
== MAP_FAILED
) {
730 new_addr
= h2g(host_addr
);
731 prot
= page_get_flags(old_addr
);
732 page_set_flags(old_addr
, old_addr
+ old_size
, 0);
733 page_set_flags(new_addr
, new_addr
+ new_size
, prot
| PAGE_VALID
);
735 tb_invalidate_phys_range(new_addr
, new_addr
+ new_size
);