linux-user/mmap.c

   1 /*
   2  *  mmap support for qemu
   3  *
   4  *  Copyright (c) 2003 Fabrice Bellard
   5  *
   6  *  This program is free software; you can redistribute it and/or modify
   7  *  it under the terms of the GNU General Public License as published by
   8  *  the Free Software Foundation; either version 2 of the License, or
   9  *  (at your option) any later version.
  10  *
  11  *  This program is distributed in the hope that it will be useful,
  12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  *  GNU General Public License for more details.
  15  *
  16  *  You should have received a copy of the GNU General Public License
  17  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
  18  */
  19 #include <stdlib.h>
  20 #include <stdio.h>
  21 #include <stdarg.h>
  22 #include <string.h>
  23 #include <unistd.h>
  24 #include <errno.h>
  25 #include <sys/types.h>
  26 #include <sys/stat.h>
  27 #include <sys/mman.h>
  28 #include <linux/mman.h>
  29 #include <linux/unistd.h>
  30
  31 #include "qemu.h"
  32 #include "qemu-common.h"
  33
  34 //#define DEBUG_MMAP
  35
  36 #if defined(CONFIG_USE_NPTL)
  37 static pthread_mutex_t mmap_mutex = PTHREAD_MUTEX_INITIALIZER;
  38 static __thread int mmap_lock_count;
  39
  40 void mmap_lock(void)
  41 {
  42     if (mmap_lock_count++ == 0) {
  43         pthread_mutex_lock(&mmap_mutex);
  44     }
  45 }
  46
  47 void mmap_unlock(void)
  48 {
  49     if (--mmap_lock_count == 0) {
  50         pthread_mutex_unlock(&mmap_mutex);
  51     }
  52 }
  53
  54 /* Grab lock to make sure things are in a consistent state after fork().  */
  55 void mmap_fork_start(void)
  56 {
  57     if (mmap_lock_count)
  58         abort();
  59     pthread_mutex_lock(&mmap_mutex);
  60 }
  61
  62 void mmap_fork_end(int child)
  63 {
  64     if (child)
  65         pthread_mutex_init(&mmap_mutex, NULL);
  66     else
  67         pthread_mutex_unlock(&mmap_mutex);
  68 }
  69 #else
  70 /* We aren't threadsafe to start with, so no need to worry about locking.  */
  71 void mmap_lock(void)
  72 {
  73 }
  74
  75 void mmap_unlock(void)
  76 {
  77 }
  78 #endif
  79
  80 void *qemu_vmalloc(size_t size)
  81 {
  82     void *p;
  83
  84     mmap_lock();
  85     /* Use map and mark the pages as used.  */
  86     p = mmap(NULL, size, PROT_READ | PROT_WRITE,
  87              MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
  88
  89     if (h2g_valid(p)) {
  90         /* Allocated region overlaps guest address space. This may recurse.  */
  91         unsigned long addr = h2g(p);
  92         page_set_flags(addr & TARGET_PAGE_MASK, TARGET_PAGE_ALIGN(addr + size),
  93                        PAGE_RESERVED);
  94     }
  95
  96     mmap_unlock();
  97     return p;
  98 }
  99
 100 void *qemu_malloc(size_t size)
 101 {
 102     char * p;
 103     size += 16;
 104     p = qemu_vmalloc(size);
 105     *(size_t *)p = size;
 106     return p + 16;
 107 }
 108
 109 /* We use map, which is always zero initialized.  */
 110 void * qemu_mallocz(size_t size)
 111 {
 112     return qemu_malloc(size);
 113 }
 114
 115 void qemu_free(void *ptr)
 116 {
 117     /* FIXME: We should unmark the reserved pages here.  However this gets
 118        complicated when one target page spans multiple host pages, so we
 119        don't bother.  */
 120     size_t *p;
 121     p = (size_t *)((char *)ptr - 16);
 122     munmap(p, *p);
 123 }
 124
 125 void *qemu_realloc(void *ptr, size_t size)
 126 {
 127     size_t old_size, copy;
 128     void *new_ptr;
 129
 130     if (!ptr)
 131         return qemu_malloc(size);
 132     old_size = *(size_t *)((char *)ptr - 16);
 133     copy = old_size < size ? old_size : size;
 134     new_ptr = qemu_malloc(size);
 135     memcpy(new_ptr, ptr, copy);
 136     qemu_free(ptr);
 137     return new_ptr;
 138 }
 139
 140 /* NOTE: all the constants are the HOST ones, but addresses are target. */
 141 int target_mprotect(abi_ulong start, abi_ulong len, int prot)
 142 {
 143     abi_ulong end, host_start, host_end, addr;
 144     int prot1, ret;
 145
 146 #ifdef DEBUG_MMAP
 147     printf("mprotect: start=0x" TARGET_ABI_FMT_lx
 148            "len=0x" TARGET_ABI_FMT_lx " prot=%c%c%c\n", start, len,
 149            prot & PROT_READ ? 'r' : '-',
 150            prot & PROT_WRITE ? 'w' : '-',
 151            prot & PROT_EXEC ? 'x' : '-');
 152 #endif
 153
 154     if ((start & ~TARGET_PAGE_MASK) != 0)
 155         return -EINVAL;
 156     len = TARGET_PAGE_ALIGN(len);
 157     end = start + len;
 158     if (end < start)
 159         return -EINVAL;
 160     prot &= PROT_READ | PROT_WRITE | PROT_EXEC;
 161     if (len == 0)
 162         return 0;
 163
 164     mmap_lock();
 165     host_start = start & qemu_host_page_mask;
 166     host_end = HOST_PAGE_ALIGN(end);
 167     if (start > host_start) {
 168         /* handle host page containing start */
 169         prot1 = prot;
 170         for(addr = host_start; addr < start; addr += TARGET_PAGE_SIZE) {
 171             prot1 |= page_get_flags(addr);
 172         }
 173         if (host_end == host_start + qemu_host_page_size) {
 174             for(addr = end; addr < host_end; addr += TARGET_PAGE_SIZE) {
 175                 prot1 |= page_get_flags(addr);
 176             }
 177             end = host_end;
 178         }
 179         ret = mprotect(g2h(host_start), qemu_host_page_size, prot1 & PAGE_BITS);
 180         if (ret != 0)
 181             goto error;
 182         host_start += qemu_host_page_size;
 183     }
 184     if (end < host_end) {
 185         prot1 = prot;
 186         for(addr = end; addr < host_end; addr += TARGET_PAGE_SIZE) {
 187             prot1 |= page_get_flags(addr);
 188         }
 189         ret = mprotect(g2h(host_end - qemu_host_page_size), qemu_host_page_size,
 190                        prot1 & PAGE_BITS);
 191         if (ret != 0)
 192             goto error;
 193         host_end -= qemu_host_page_size;
 194     }
 195
 196     /* handle the pages in the middle */
 197     if (host_start < host_end) {
 198         ret = mprotect(g2h(host_start), host_end - host_start, prot);
 199         if (ret != 0)
 200             goto error;
 201     }
 202     page_set_flags(start, start + len, prot | PAGE_VALID);
 203     mmap_unlock();
 204     return 0;
 205 error:
 206     mmap_unlock();
 207     return ret;
 208 }
 209
 210 /* map an incomplete host page */
 211 static int mmap_frag(abi_ulong real_start,
 212                      abi_ulong start, abi_ulong end,
 213                      int prot, int flags, int fd, abi_ulong offset)
 214 {
 215     abi_ulong real_end, addr;
 216     void *host_start;
 217     int prot1, prot_new;
 218
 219     real_end = real_start + qemu_host_page_size;
 220     host_start = g2h(real_start);
 221
 222     /* get the protection of the target pages outside the mapping */
 223     prot1 = 0;
 224     for(addr = real_start; addr < real_end; addr++) {
 225         if (addr < start || addr >= end)
 226             prot1 |= page_get_flags(addr);
 227     }
 228
 229     if (prot1 == 0) {
 230         /* no page was there, so we allocate one */
 231         void *p = mmap(host_start, qemu_host_page_size, prot,
 232                        flags | MAP_ANONYMOUS, -1, 0);
 233         if (p == MAP_FAILED)
 234             return -1;
 235         prot1 = prot;
 236     }
 237     prot1 &= PAGE_BITS;
 238
 239     prot_new = prot | prot1;
 240     if (!(flags & MAP_ANONYMOUS)) {
 241         /* msync() won't work here, so we return an error if write is
 242            possible while it is a shared mapping */
 243         if ((flags & MAP_TYPE) == MAP_SHARED &&
 244             (prot & PROT_WRITE))
 245             return -1;
 246
 247         /* adjust protection to be able to read */
 248         if (!(prot1 & PROT_WRITE))
 249             mprotect(host_start, qemu_host_page_size, prot1 | PROT_WRITE);
 250
 251         /* read the corresponding file data */
 252         if (pread(fd, g2h(start), end - start, offset) == -1)
 253             return -1;
 254
 255         /* put final protection */
 256         if (prot_new != (prot1 | PROT_WRITE))
 257             mprotect(host_start, qemu_host_page_size, prot_new);
 258     } else {
 259         /* just update the protection */
 260         if (prot_new != prot1) {
 261             mprotect(host_start, qemu_host_page_size, prot_new);
 262         }
 263     }
 264     return 0;
 265 }
 266
 267 #if defined(__CYGWIN__)
 268 /* Cygwin doesn't have a whole lot of address space.  */
 269 static abi_ulong mmap_next_start = 0x18000000;
 270 #else
 271 static abi_ulong mmap_next_start = 0x40000000;
 272 #endif
 273
 274 unsigned long last_brk;
 275
 276 /*
 277  * Find and reserve a free memory area of size 'size'. The search
 278  * starts at 'start'.
 279  * It must be called with mmap_lock() held.
 280  * Return -1 if error.
 281  */
 282 abi_ulong mmap_find_vma(abi_ulong start, abi_ulong size)
 283 {
 284     void *ptr;
 285     abi_ulong addr;
 286
 287     size = HOST_PAGE_ALIGN(size);
 288     start &= qemu_host_page_mask;
 289
 290     /* If 'start' == 0, then a default start address is used. */
 291     if (start == 0)
 292         start = mmap_next_start;
 293
 294     addr = start;
 295
 296     for(;;) {
 297         /*
 298          * Reserve needed memory area to avoid a race.
 299          * It should be discarded using:
 300          *  - mmap() with MAP_FIXED flag
 301          *  - mremap() with MREMAP_FIXED flag
 302          *  - shmat() with SHM_REMAP flag
 303          */
 304         ptr = mmap((void *)(unsigned long)addr, size, PROT_NONE,
 305                    MAP_ANONYMOUS|MAP_PRIVATE|MAP_NORESERVE, -1, 0);
 306
 307         /* ENOMEM, if host address space has no memory */
 308         if (ptr == MAP_FAILED)
 309             return (abi_ulong)-1;
 310
 311         /* If address fits target address space we've found what we need */
 312         if ((unsigned long)ptr + size - 1 <= (abi_ulong)-1)
 313             break;
 314
 315         /* Unmap and try again with new page */
 316         munmap(ptr, size);
 317         addr += qemu_host_page_size;
 318
 319         /* ENOMEM if we check whole of target address space */
 320         if (addr == start)
 321             return (abi_ulong)-1;
 322     }
 323
 324     /* Update default start address */
 325     if (start == mmap_next_start)
 326         mmap_next_start = (unsigned long)ptr + size;
 327
 328     return h2g(ptr);
 329 }
 330
 331 /* NOTE: all the constants are the HOST ones */
 332 abi_long target_mmap(abi_ulong start, abi_ulong len, int prot,
 333                      int flags, int fd, abi_ulong offset)
 334 {
 335     abi_ulong ret, end, real_start, real_end, retaddr, host_offset, host_len;
 336     unsigned long host_start;
 337
 338     mmap_lock();
 339 #ifdef DEBUG_MMAP
 340     {
 341         printf("mmap: start=0x" TARGET_ABI_FMT_lx
 342                " len=0x" TARGET_ABI_FMT_lx " prot=%c%c%c flags=",
 343                start, len,
 344                prot & PROT_READ ? 'r' : '-',
 345                prot & PROT_WRITE ? 'w' : '-',
 346                prot & PROT_EXEC ? 'x' : '-');
 347         if (flags & MAP_FIXED)
 348             printf("MAP_FIXED ");
 349         if (flags & MAP_ANONYMOUS)
 350             printf("MAP_ANON ");
 351         switch(flags & MAP_TYPE) {
 352         case MAP_PRIVATE:
 353             printf("MAP_PRIVATE ");
 354             break;
 355         case MAP_SHARED:
 356             printf("MAP_SHARED ");
 357             break;
 358         default:
 359             printf("[MAP_TYPE=0x%x] ", flags & MAP_TYPE);
 360             break;
 361         }
 362         printf("fd=%d offset=" TARGET_ABI_FMT_lx "\n", fd, offset);
 363     }
 364 #endif
 365
 366     if (offset & ~TARGET_PAGE_MASK) {
 367         errno = EINVAL;
 368         goto fail;
 369     }
 370
 371     len = TARGET_PAGE_ALIGN(len);
 372     if (len == 0)
 373         goto the_end;
 374     real_start = start & qemu_host_page_mask;
 375
 376     /* When mapping files into a memory area larger than the file, accesses
 377        to pages beyond the file size will cause a SIGBUS.
 378
 379        For example, if mmaping a file of 100 bytes on a host with 4K pages
 380        emulating a target with 8K pages, the target expects to be able to
 381        access the first 8K. But the host will trap us on any access beyond
 382        4K.
 383
 384        When emulating a target with a larger page-size than the hosts, we
 385        may need to truncate file maps at EOF and add extra anonymous pages
 386        up to the targets page boundary.  */
 387
 388     if ((qemu_real_host_page_size < TARGET_PAGE_SIZE)
 389         && !(flags & MAP_ANONYMOUS)) {
 390        struct stat sb;
 391
 392        if (fstat (fd, &sb) == -1)
 393            goto fail;
 394
 395        /* Are we trying to create a map beyond EOF?.  */
 396        if (offset + len > sb.st_size) {
 397            /* If so, truncate the file map at eof aligned with
 398               the hosts real pagesize. Additional anonymous maps
 399               will be created beyond EOF.  */
 400            len = (sb.st_size - offset);
 401            len += qemu_real_host_page_size - 1;
 402            len &= ~(qemu_real_host_page_size - 1);
 403        }
 404     }
 405
 406     if (!(flags & MAP_FIXED)) {
 407         abi_ulong mmap_start;
 408         void *p;
 409         host_offset = offset & qemu_host_page_mask;
 410         host_len = len + offset - host_offset;
 411         host_len = HOST_PAGE_ALIGN(host_len);
 412         mmap_start = mmap_find_vma(real_start, host_len);
 413         if (mmap_start == (abi_ulong)-1) {
 414             errno = ENOMEM;
 415             goto fail;
 416         }
 417         /* Note: we prefer to control the mapping address. It is
 418            especially important if qemu_host_page_size >
 419            qemu_real_host_page_size */
 420         p = mmap(g2h(mmap_start),
 421                  host_len, prot, flags | MAP_FIXED | MAP_ANONYMOUS, -1, 0);
 422         if (p == MAP_FAILED)
 423             goto fail;
 424         /* update start so that it points to the file position at 'offset' */
 425         host_start = (unsigned long)p;
 426         if (!(flags & MAP_ANONYMOUS)) {
 427             p = mmap(g2h(mmap_start), len, prot,
 428                      flags | MAP_FIXED, fd, host_offset);
 429             host_start += offset - host_offset;
 430         }
 431         start = h2g(host_start);
 432     } else {
 433         int flg;
 434         target_ulong addr;
 435
 436         if (start & ~TARGET_PAGE_MASK) {
 437             errno = EINVAL;
 438             goto fail;
 439         }
 440         end = start + len;
 441         real_end = HOST_PAGE_ALIGN(end);
 442
 443         /*
 444          * Test if requested memory area fits target address space
 445          * It can fail only on 64-bit host with 32-bit target.
 446          * On any other target/host host mmap() handles this error correctly.
 447          */
 448         if ((unsigned long)start + len - 1 > (abi_ulong) -1) {
 449             errno = EINVAL;
 450             goto fail;
 451         }
 452
 453         for(addr = real_start; addr < real_end; addr += TARGET_PAGE_SIZE) {
 454             flg = page_get_flags(addr);
 455             if (flg & PAGE_RESERVED) {
 456                 errno = ENXIO;
 457                 goto fail;
 458             }
 459         }
 460
 461         /* worst case: we cannot map the file because the offset is not
 462            aligned, so we read it */
 463         if (!(flags & MAP_ANONYMOUS) &&
 464             (offset & ~qemu_host_page_mask) != (start & ~qemu_host_page_mask)) {
 465             /* msync() won't work here, so we return an error if write is
 466                possible while it is a shared mapping */
 467             if ((flags & MAP_TYPE) == MAP_SHARED &&
 468                 (prot & PROT_WRITE)) {
 469                 errno = EINVAL;
 470                 goto fail;
 471             }
 472             retaddr = target_mmap(start, len, prot | PROT_WRITE,
 473                                   MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS,
 474                                   -1, 0);
 475             if (retaddr == -1)
 476                 goto fail;
 477             if (pread(fd, g2h(start), len, offset) == -1)
 478                 goto fail;
 479             if (!(prot & PROT_WRITE)) {
 480                 ret = target_mprotect(start, len, prot);
 481                 if (ret != 0) {
 482                     start = ret;
 483                     goto the_end;
 484                 }
 485             }
 486             goto the_end;
 487         }
 488
 489         /* handle the start of the mapping */
 490         if (start > real_start) {
 491             if (real_end == real_start + qemu_host_page_size) {
 492                 /* one single host page */
 493                 ret = mmap_frag(real_start, start, end,
 494                                 prot, flags, fd, offset);
 495                 if (ret == -1)
 496                     goto fail;
 497                 goto the_end1;
 498             }
 499             ret = mmap_frag(real_start, start, real_start + qemu_host_page_size,
 500                             prot, flags, fd, offset);
 501             if (ret == -1)
 502                 goto fail;
 503             real_start += qemu_host_page_size;
 504         }
 505         /* handle the end of the mapping */
 506         if (end < real_end) {
 507             ret = mmap_frag(real_end - qemu_host_page_size,
 508                             real_end - qemu_host_page_size, real_end,
 509                             prot, flags, fd,
 510                             offset + real_end - qemu_host_page_size - start);
 511             if (ret == -1)
 512                 goto fail;
 513             real_end -= qemu_host_page_size;
 514         }
 515
 516         /* map the middle (easier) */
 517         if (real_start < real_end) {
 518             void *p;
 519             unsigned long offset1;
 520             if (flags & MAP_ANONYMOUS)
 521                 offset1 = 0;
 522             else
 523                 offset1 = offset + real_start - start;
 524             p = mmap(g2h(real_start), real_end - real_start,
 525                      prot, flags, fd, offset1);
 526             if (p == MAP_FAILED)
 527                 goto fail;
 528         }
 529     }
 530  the_end1:
 531     page_set_flags(start, start + len, prot | PAGE_VALID);
 532  the_end:
 533 #ifdef DEBUG_MMAP
 534     printf("ret=0x" TARGET_ABI_FMT_lx "\n", start);
 535     page_dump(stdout);
 536     printf("\n");
 537 #endif
 538     mmap_unlock();
 539     return start;
 540 fail:
 541     mmap_unlock();
 542     return -1;
 543 }
 544
 545 int target_munmap(abi_ulong start, abi_ulong len)
 546 {
 547     abi_ulong end, real_start, real_end, addr;
 548     int prot, ret;
 549
 550 #ifdef DEBUG_MMAP
 551     printf("munmap: start=0x" TARGET_ABI_FMT_lx " len=0x"
 552            TARGET_ABI_FMT_lx "\n",
 553            start, len);
 554 #endif
 555     if (start & ~TARGET_PAGE_MASK)
 556         return -EINVAL;
 557     len = TARGET_PAGE_ALIGN(len);
 558     if (len == 0)
 559         return -EINVAL;
 560     mmap_lock();
 561     end = start + len;
 562     real_start = start & qemu_host_page_mask;
 563     real_end = HOST_PAGE_ALIGN(end);
 564
 565     if (start > real_start) {
 566         /* handle host page containing start */
 567         prot = 0;
 568         for(addr = real_start; addr < start; addr += TARGET_PAGE_SIZE) {
 569             prot |= page_get_flags(addr);
 570         }
 571         if (real_end == real_start + qemu_host_page_size) {
 572             for(addr = end; addr < real_end; addr += TARGET_PAGE_SIZE) {
 573                 prot |= page_get_flags(addr);
 574             }
 575             end = real_end;
 576         }
 577         if (prot != 0)
 578             real_start += qemu_host_page_size;
 579     }
 580     if (end < real_end) {
 581         prot = 0;
 582         for(addr = end; addr < real_end; addr += TARGET_PAGE_SIZE) {
 583             prot |= page_get_flags(addr);
 584         }
 585         if (prot != 0)
 586             real_end -= qemu_host_page_size;
 587     }
 588
 589     ret = 0;
 590     /* unmap what we can */
 591     if (real_start < real_end) {
 592         ret = munmap(g2h(real_start), real_end - real_start);
 593     }
 594
 595     if (ret == 0)
 596         page_set_flags(start, start + len, 0);
 597     mmap_unlock();
 598     return ret;
 599 }
 600
 601 abi_long target_mremap(abi_ulong old_addr, abi_ulong old_size,
 602                        abi_ulong new_size, unsigned long flags,
 603                        abi_ulong new_addr)
 604 {
 605     int prot;
 606     void *host_addr;
 607
 608     mmap_lock();
 609
 610     if (flags & MREMAP_FIXED)
 611         host_addr = (void *) syscall(__NR_mremap, g2h(old_addr),
 612                                      old_size, new_size,
 613                                      flags,
 614                                      new_addr);
 615     else if (flags & MREMAP_MAYMOVE) {
 616         abi_ulong mmap_start;
 617
 618         mmap_start = mmap_find_vma(0, new_size);
 619
 620         if (mmap_start == -1) {
 621             errno = ENOMEM;
 622             host_addr = MAP_FAILED;
 623         } else
 624             host_addr = (void *) syscall(__NR_mremap, g2h(old_addr),
 625                                          old_size, new_size,
 626                                          flags | MREMAP_FIXED,
 627                                          g2h(mmap_start));
 628     } else {
 629         host_addr = mremap(g2h(old_addr), old_size, new_size, flags);
 630         /* Check if address fits target address space */
 631         if ((unsigned long)host_addr + new_size > (abi_ulong)-1) {
 632             /* Revert mremap() changes */
 633             host_addr = mremap(g2h(old_addr), new_size, old_size, flags);
 634             errno = ENOMEM;
 635             host_addr = MAP_FAILED;
 636         }
 637     }
 638
 639     if (host_addr == MAP_FAILED) {
 640         new_addr = -1;
 641     } else {
 642         new_addr = h2g(host_addr);
 643         prot = page_get_flags(old_addr);
 644         page_set_flags(old_addr, old_addr + old_size, 0);
 645         page_set_flags(new_addr, new_addr + new_size, prot | PAGE_VALID);
 646     }
 647     mmap_unlock();
 648     return new_addr;
 649 }
 650
 651 int target_msync(abi_ulong start, abi_ulong len, int flags)
 652 {
 653     abi_ulong end;
 654
 655     if (start & ~TARGET_PAGE_MASK)
 656         return -EINVAL;
 657     len = TARGET_PAGE_ALIGN(len);
 658     end = start + len;
 659     if (end < start)
 660         return -EINVAL;
 661     if (end == start)
 662         return 0;
 663
 664     start &= qemu_host_page_mask;
 665     return msync(g2h(start), end - start, flags);
 666 }