ceph/src/pmdk/src/windows/win_mmap.c

   1 // SPDX-License-Identifier: BSD-3-Clause
   2 /* Copyright 2015-2019, Intel Corporation */
   3 /*
   4  * Copyright (c) 2015-2017, Microsoft Corporation. All rights reserved.
   5  * Copyright (c) 2016, Hewlett Packard Enterprise Development LP
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *
  14  *     * Redistributions in binary form must reproduce the above copyright
  15  *       notice, this list of conditions and the following disclaimer in
  16  *       the documentation and/or other materials provided with the
  17  *       distribution.
  18  *
  19  *     * Neither the name of the copyright holder nor the names of its
  20  *       contributors may be used to endorse or promote products derived
  21  *       from this software without specific prior written permission.
  22  *
  23  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  24  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  25  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  26  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  27  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  28  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  29  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  30  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  31  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  32  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  33  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  34  */
  35
  36 /*
  37  * win_mmap.c -- memory-mapped files for Windows
  38  */
  39
  40 /*
  41  * XXX - The initial approach to PMDK for Windows port was to minimize the
  42  * amount of changes required in the core part of the library, and to avoid
  43  * preprocessor conditionals, if possible.  For that reason, some of the
  44  * Linux system calls that have no equivalents on Windows have been emulated
  45  * using Windows API.
  46  * Note that it was not a goal to fully emulate POSIX-compliant behavior
  47  * of mentioned functions.  They are used only internally, so current
  48  * implementation is just good enough to satisfy PMDK needs and to make it
  49  * work on Windows.
  50  *
  51  * This is a subject for change in the future.  Likely, all these functions
  52  * will be replaced with "util_xxx" wrappers with OS-specific implementation
  53  * for Linux and Windows.
  54  *
  55  * Known issues:
  56  * - on Windows, mapping granularity/alignment is 64KB, not 4KB;
  57  * - mprotect() behavior and protection flag handling in mmap() is slightly
  58  *   different than on Linux (see comments below).
  59  */
  60
  61 #include <sys/mman.h>
  62 #include "mmap.h"
  63 #include "util.h"
  64 #include "out.h"
  65 #include "win_mmap.h"
  66
  67 /* uncomment for more debug information on mmap trackers */
  68 /* #define MMAP_DEBUG_INFO */
  69
  70 NTSTATUS
  71 NtFreeVirtualMemory(_In_ HANDLE ProcessHandle, _Inout_ PVOID *BaseAddress,
  72         _Inout_ PSIZE_T RegionSize, _In_ ULONG FreeType);
  73
  74 /*
  75  * XXX Unify the Linux and Windows code and replace this structure with
  76  * the map tracking list defined in mmap.h.
  77  */
  78 SRWLOCK FileMappingQLock = SRWLOCK_INIT;
  79 struct FMLHead FileMappingQHead =
  80         PMDK_SORTEDQ_HEAD_INITIALIZER(FileMappingQHead);
  81
  82 /*
  83  * mmap_file_mapping_comparer -- (internal) compares the two file mapping
  84  * trackers
  85  */
  86 static LONG_PTR
  87 mmap_file_mapping_comparer(PFILE_MAPPING_TRACKER a, PFILE_MAPPING_TRACKER b)
  88 {
  89         return ((LONG_PTR)a->BaseAddress - (LONG_PTR)b->BaseAddress);
  90 }
  91
  92 #ifdef MMAP_DEBUG_INFO
  93 /*
  94  * mmap_info -- (internal) dump info about all the mapping trackers
  95  */
  96 static void
  97 mmap_info(void)
  98 {
  99         LOG(4, NULL);
 100
 101         AcquireSRWLockShared(&FileMappingQLock);
 102
 103         PFILE_MAPPING_TRACKER mt;
 104         for (mt = PMDK_SORTEDQ_FIRST(&FileMappingQHead);
 105                 mt != (void *)&FileMappingQHead;
 106                 mt = PMDK_SORTEDQ_NEXT(mt, ListEntry)) {
 107
 108                 LOG(4, "FH %08x FMH %08x AD %p-%p (%zu) "
 109                         "OF %08x FL %zu AC %d F %d",
 110                         mt->FileHandle,
 111                         mt->FileMappingHandle,
 112                         mt->BaseAddress,
 113                         mt->EndAddress,
 114                         (char *)mt->EndAddress - (char *)mt->BaseAddress,
 115                         mt->Offset,
 116                         mt->FileLen,
 117                         mt->Access,
 118                         mt->Flags);
 119         }
 120
 121         ReleaseSRWLockShared(&FileMappingQLock);
 122 }
 123 #endif
 124
 125 /*
 126  * mmap_reserve -- (internal) reserve virtual address range
 127  */
 128 static void *
 129 mmap_reserve(void *addr, size_t len)
 130 {
 131         LOG(4, "addr %p len %zu", addr, len);
 132
 133         ASSERTeq((uintptr_t)addr % Mmap_align, 0);
 134         ASSERTeq(len % Mmap_align, 0);
 135
 136         void *reserved_addr = VirtualAlloc(addr, len,
 137                                 MEM_RESERVE, PAGE_NOACCESS);
 138         if (reserved_addr == NULL) {
 139                 ERR("cannot find a contiguous region - "
 140                         "addr: %p, len: %lx, gle: 0x%08x",
 141                         addr, len, GetLastError());
 142                 errno = ENOMEM;
 143                 return MAP_FAILED;
 144         }
 145
 146         return reserved_addr;
 147 }
 148
 149 /*
 150  * mmap_unreserve -- (internal) frees the range that's previously reserved
 151  */
 152 static int
 153 mmap_unreserve(void *addr, size_t len)
 154 {
 155         LOG(4, "addr %p len %zu", addr, len);
 156
 157         ASSERTeq((uintptr_t)addr % Mmap_align, 0);
 158         ASSERTeq(len % Mmap_align, 0);
 159
 160         size_t bytes_returned;
 161         MEMORY_BASIC_INFORMATION basic_info;
 162
 163         bytes_returned = VirtualQuery(addr, &basic_info, sizeof(basic_info));
 164
 165         if (bytes_returned != sizeof(basic_info)) {
 166                 ERR("cannot query the virtual address properties of the range "
 167                         "- addr: %p, len: %d", addr, len);
 168                 errno = EINVAL;
 169                 return -1;
 170         }
 171
 172         if (basic_info.State == MEM_RESERVE) {
 173                 DWORD nt_status;
 174                 void *release_addr = addr;
 175                 size_t release_size = len;
 176                 nt_status = NtFreeVirtualMemory(GetCurrentProcess(),
 177                         &release_addr, &release_size, MEM_RELEASE);
 178                 if (nt_status != 0) {
 179                         ERR("cannot release the reserved virtual space - "
 180                                 "addr: %p, len: %d, nt_status: 0x%08x",
 181                                 addr, len, nt_status);
 182                         errno = EINVAL;
 183                         return -1;
 184                 }
 185                 ASSERTeq(release_addr, addr);
 186                 ASSERTeq(release_size, len);
 187                 LOG(4, "freed reservation - addr: %p, size: %d", release_addr,
 188                         release_size);
 189         } else {
 190                 LOG(4, "range not reserved - addr: %p, size: %d", addr, len);
 191         }
 192
 193         return 0;
 194 }
 195
 196 /*
 197  * win_mmap_init -- initialization of file mapping tracker
 198  */
 199 void
 200 win_mmap_init(void)
 201 {
 202         AcquireSRWLockExclusive(&FileMappingQLock);
 203         PMDK_SORTEDQ_INIT(&FileMappingQHead);
 204         ReleaseSRWLockExclusive(&FileMappingQLock);
 205 }
 206
 207 /*
 208  * win_mmap_fini -- file mapping tracker cleanup routine
 209  */
 210 void
 211 win_mmap_fini(void)
 212 {
 213         /*
 214          * Let's make sure that no one is in the middle of updating the
 215          * list by grabbing the lock.
 216          */
 217         AcquireSRWLockExclusive(&FileMappingQLock);
 218
 219         while (!PMDK_SORTEDQ_EMPTY(&FileMappingQHead)) {
 220                 PFILE_MAPPING_TRACKER mt;
 221                 mt = (PFILE_MAPPING_TRACKER)PMDK_SORTEDQ_FIRST(
 222                                 &FileMappingQHead);
 223
 224                 PMDK_SORTEDQ_REMOVE(&FileMappingQHead, mt, ListEntry);
 225
 226                 if (mt->BaseAddress != NULL)
 227                         UnmapViewOfFile(mt->BaseAddress);
 228
 229                 size_t release_size =
 230                         (char *)mt->EndAddress - (char *)mt->BaseAddress;
 231                 /*
 232                  * Free reservation after file mapping (if reservation was
 233                  * bigger than length of mapped file)
 234                  */
 235                 void *release_addr = (char *)mt->BaseAddress + mt->FileLen;
 236                 mmap_unreserve(release_addr, release_size - mt->FileLen);
 237
 238                 if (mt->FileMappingHandle != NULL)
 239                         CloseHandle(mt->FileMappingHandle);
 240
 241                 if (mt->FileHandle != NULL)
 242                         CloseHandle(mt->FileHandle);
 243
 244                 free(mt);
 245         }
 246         ReleaseSRWLockExclusive(&FileMappingQLock);
 247 }
 248
 249 #define PROT_ALL (PROT_READ|PROT_WRITE|PROT_EXEC)
 250
 251 /*
 252  * mmap -- map file into memory
 253  *
 254  * XXX - If read-only mapping was created initially, it is not possible
 255  * to change protection to R/W, even if the file itself was open in R/W mode.
 256  * To workaround that, we could modify mmap() to create R/W mapping first,
 257  * then change the protection to R/O.  This way, it should be possible
 258  * to elevate permissions later.
 259  */
 260 void *
 261 mmap(void *addr, size_t len, int prot, int flags, int fd, os_off_t offset)
 262 {
 263         LOG(4, "addr %p len %zu prot %d flags %d fd %d offset %ju",
 264                 addr, len, prot, flags, fd, offset);
 265
 266         if (len == 0) {
 267                 ERR("invalid length: %zu", len);
 268                 errno = EINVAL;
 269                 return MAP_FAILED;
 270         }
 271
 272         if ((prot & ~PROT_ALL) != 0) {
 273                 ERR("invalid flags: 0x%08x", flags);
 274                 /* invalid protection flags */
 275                 errno = EINVAL;
 276                 return MAP_FAILED;
 277         }
 278
 279         if (((flags & MAP_PRIVATE) && (flags & MAP_SHARED)) ||
 280             ((flags & (MAP_PRIVATE | MAP_SHARED)) == 0)) {
 281                 ERR("neither MAP_PRIVATE or MAP_SHARED is set, or both: 0x%08x",
 282                         flags);
 283                 errno = EINVAL;
 284                 return MAP_FAILED;
 285         }
 286
 287         /* XXX shall we use SEC_LARGE_PAGES flag? */
 288         DWORD protect = 0;
 289         DWORD access = 0;
 290
 291         /* on x86, PROT_WRITE implies PROT_READ */
 292         if (prot & PROT_WRITE) {
 293                 if (flags & MAP_PRIVATE) {
 294                         access = FILE_MAP_COPY;
 295                         if (prot & PROT_EXEC)
 296                                 protect = PAGE_EXECUTE_WRITECOPY;
 297                         else
 298                                 protect = PAGE_WRITECOPY;
 299                 } else {
 300                         /* FILE_MAP_ALL_ACCESS == FILE_MAP_WRITE */
 301                         access = FILE_MAP_ALL_ACCESS;
 302                         if (prot & PROT_EXEC)
 303                                 protect = PAGE_EXECUTE_READWRITE;
 304                         else
 305                                 protect = PAGE_READWRITE;
 306                 }
 307         } else if (prot & PROT_READ) {
 308                 access = FILE_MAP_READ;
 309                 if (prot & PROT_EXEC)
 310                         protect = PAGE_EXECUTE_READ;
 311                 else
 312                         protect = PAGE_READONLY;
 313         } else {
 314                 /* XXX - PAGE_NOACCESS is not supported by CreateFileMapping */
 315                 ERR("PAGE_NOACCESS is not supported");
 316                 errno = ENOTSUP;
 317                 return MAP_FAILED;
 318         }
 319
 320         if (((uintptr_t)addr % Mmap_align) != 0) {
 321                 if ((flags & MAP_FIXED) == 0) {
 322                         /* ignore invalid hint if no MAP_FIXED flag is set */
 323                         addr = NULL;
 324                 } else {
 325                         ERR("hint address is not well-aligned: %p", addr);
 326                         errno = EINVAL;
 327                         return MAP_FAILED;
 328                 }
 329         }
 330
 331         if ((offset % Mmap_align) != 0) {
 332                 ERR("offset is not well-aligned: %ju", offset);
 333                 errno = EINVAL;
 334                 return MAP_FAILED;
 335         }
 336
 337         if ((flags & MAP_FIXED) != 0) {
 338                 /*
 339                  * Free any reservations that the caller might have, also we
 340                  * have to unmap any existing mappings in this region as per
 341                  * mmap's manual.
 342                  * XXX - Ideally we should unmap only if the prot and flags
 343                  * are similar, we are deferring it as we don't rely on it
 344                  * yet.
 345                  */
 346                 int ret = munmap(addr, len);
 347                 if (ret != 0) {
 348                         ERR("!munmap: addr %p len %zu", addr, len);
 349                         return MAP_FAILED;
 350                 }
 351         }
 352
 353         size_t len_align = roundup(len, Mmap_align);
 354         size_t filelen;
 355         size_t filelen_align;
 356         HANDLE fh;
 357         if (flags & MAP_ANON) {
 358                 /*
 359                  * In our implementation we are choosing to ignore fd when
 360                  * MAP_ANON is set, instead of failing.
 361                  */
 362                 fh = INVALID_HANDLE_VALUE;
 363
 364                 /* ignore/override offset */
 365                 offset = 0;
 366                 filelen = len;
 367                 filelen_align = len_align;
 368
 369                 if ((flags & MAP_NORESERVE) != 0) {
 370                         /*
 371                          * For anonymous mappings the meaning of MAP_NORESERVE
 372                          * flag is pretty much the same as SEC_RESERVE.
 373                          */
 374                         protect |= SEC_RESERVE;
 375                 }
 376         } else {
 377                 LARGE_INTEGER filesize;
 378
 379                 if (fd == -1) {
 380                         ERR("invalid file descriptor: %d", fd);
 381                         errno = EBADF;
 382                         return MAP_FAILED;
 383                 }
 384
 385                 /*
 386                  * We need to keep file handle open for proper
 387                  * implementation of msync() and to hold the file lock.
 388                  */
 389                 if (!DuplicateHandle(GetCurrentProcess(),
 390                                 (HANDLE)_get_osfhandle(fd),
 391                                 GetCurrentProcess(), &fh,
 392                                 0, FALSE, DUPLICATE_SAME_ACCESS)) {
 393                         ERR("cannot duplicate handle - fd: %d, gle: 0x%08x",
 394                                         fd, GetLastError());
 395                         errno = ENOMEM;
 396                         return MAP_FAILED;
 397                 }
 398
 399                 /*
 400                  * If we are asked to map more than the file size, map till the
 401                  * file size and reserve the following.
 402                  */
 403
 404                 if (!GetFileSizeEx(fh, &filesize)) {
 405                         ERR("cannot query the file size - fh: %d, gle: 0x%08x",
 406                                 fd, GetLastError());
 407                         CloseHandle(fh);
 408                         return MAP_FAILED;
 409                 }
 410
 411                 if (offset >= (os_off_t)filesize.QuadPart) {
 412                         errno = EINVAL;
 413                         ERR("offset is beyond the file size");
 414                         CloseHandle(fh);
 415                         return MAP_FAILED;
 416                 }
 417
 418                 /* calculate length of the mapped portion of the file */
 419                 filelen = filesize.QuadPart - offset;
 420                 if (filelen > len)
 421                         filelen = len;
 422                 filelen_align = roundup(filelen, Mmap_align);
 423
 424                 if ((offset + len) > (size_t)filesize.QuadPart) {
 425                         /*
 426                          * Reserve virtual address for the rest of range we need
 427                          * to map, and free a portion in the beginning for this
 428                          * allocation.
 429                          */
 430                         void *reserved_addr = mmap_reserve(addr, len_align);
 431                         if (reserved_addr == MAP_FAILED) {
 432                                 ERR("cannot reserve region");
 433                                 CloseHandle(fh);
 434                                 return MAP_FAILED;
 435                         }
 436
 437                         if (addr != reserved_addr && (flags & MAP_FIXED) != 0) {
 438                                 ERR("cannot find a contiguous region - "
 439                                         "addr: %p, len: %lx, gle: 0x%08x",
 440                                         addr, len, GetLastError());
 441                                 if (mmap_unreserve(reserved_addr,
 442                                                 len_align) != 0) {
 443                                         ASSERT(FALSE);
 444                                         ERR("cannot free reserved region");
 445                                 }
 446                                 errno = ENOMEM;
 447                                 CloseHandle(fh);
 448                                 return MAP_FAILED;
 449                         }
 450
 451                         addr = reserved_addr;
 452                         if (mmap_unreserve(reserved_addr, filelen_align) != 0) {
 453                                 ASSERT(FALSE);
 454                                 ERR("cannot free reserved region");
 455                                 CloseHandle(fh);
 456                                 return MAP_FAILED;
 457                         }
 458                 }
 459         }
 460
 461         HANDLE fmh = CreateFileMapping(fh,
 462                         NULL, /* security attributes */
 463                         protect,
 464                         (DWORD) ((filelen + offset) >> 32),
 465                         (DWORD) ((filelen + offset) & 0xFFFFFFFF),
 466                         NULL);
 467
 468         if (fmh == NULL) {
 469                 DWORD gle = GetLastError();
 470                 ERR("CreateFileMapping, gle: 0x%08x", gle);
 471                 if (gle == ERROR_ACCESS_DENIED)
 472                         errno = EACCES;
 473                 else
 474                         errno = EINVAL; /* XXX */
 475                 CloseHandle(fh);
 476                 return MAP_FAILED;
 477         }
 478
 479         void *base = MapViewOfFileEx(fmh,
 480                         access,
 481                         (DWORD) (offset >> 32),
 482                         (DWORD) (offset & 0xFFFFFFFF),
 483                         filelen,
 484                         addr); /* hint address */
 485
 486         if (base == NULL) {
 487                 if (addr == NULL || (flags & MAP_FIXED) != 0) {
 488                         ERR("MapViewOfFileEx, gle: 0x%08x", GetLastError());
 489                         errno = EINVAL;
 490                         CloseHandle(fh);
 491                         CloseHandle(fmh);
 492                         return MAP_FAILED;
 493                 }
 494
 495                 /* try again w/o hint */
 496                 base = MapViewOfFileEx(fmh,
 497                                 access,
 498                                 (DWORD) (offset >> 32),
 499                                 (DWORD) (offset & 0xFFFFFFFF),
 500                                 filelen,
 501                                 NULL); /* no hint address */
 502         }
 503
 504         if (base == NULL) {
 505                 ERR("MapViewOfFileEx, gle: 0x%08x", GetLastError());
 506                 errno = ENOMEM;
 507                 CloseHandle(fh);
 508                 CloseHandle(fmh);
 509                 return MAP_FAILED;
 510         }
 511
 512         /*
 513          * We will track the file mapping handle on a lookaside list so that
 514          * we don't have to modify the fact that we only return back the base
 515          * address rather than a more elaborate structure.
 516          */
 517
 518         PFILE_MAPPING_TRACKER mt =
 519                 malloc(sizeof(struct FILE_MAPPING_TRACKER));
 520
 521         if (mt == NULL) {
 522                 ERR("!malloc");
 523                 CloseHandle(fh);
 524                 CloseHandle(fmh);
 525                 return MAP_FAILED;
 526         }
 527
 528         mt->Flags = 0;
 529         mt->FileHandle = fh;
 530         mt->FileMappingHandle = fmh;
 531         mt->BaseAddress = base;
 532         mt->EndAddress = (void *)((char *)base + len_align);
 533         mt->Access = access;
 534         mt->Offset = offset;
 535         mt->FileLen = filelen_align;
 536
 537         /*
 538          * XXX: Use the QueryVirtualMemoryInformation when available in the new
 539          * SDK.  If the file is DAX mapped say so in the FILE_MAPPING_TRACKER
 540          * Flags.
 541          */
 542         DWORD filesystemFlags;
 543         if (fh == INVALID_HANDLE_VALUE) {
 544                 LOG(4, "anonymous mapping - not DAX mapped - handle: %p", fh);
 545         } else if (GetVolumeInformationByHandleW(fh, NULL, 0, NULL, NULL,
 546                                 &filesystemFlags, NULL, 0)) {
 547                 if (filesystemFlags & FILE_DAX_VOLUME) {
 548                         mt->Flags |= FILE_MAPPING_TRACKER_FLAG_DIRECT_MAPPED;
 549                 } else {
 550                         LOG(4, "file is not DAX mapped - handle: %p", fh);
 551                 }
 552         } else {
 553                 ERR("failed to query volume information : %08x",
 554                         GetLastError());
 555         }
 556
 557         AcquireSRWLockExclusive(&FileMappingQLock);
 558
 559         PMDK_SORTEDQ_INSERT(&FileMappingQHead, mt, ListEntry,
 560                         FILE_MAPPING_TRACKER, mmap_file_mapping_comparer);
 561
 562         ReleaseSRWLockExclusive(&FileMappingQLock);
 563
 564 #ifdef MMAP_DEBUG_INFO
 565         mmap_info();
 566 #endif
 567
 568         return base;
 569 }
 570
 571 /*
 572  * mmap_split -- (internal) replace existing mapping with another one(s)
 573  *
 574  * Unmaps the region between [begin,end].  If it's in a middle of the existing
 575  * mapping, it results in two new mappings and duplicated file/mapping handles.
 576  */
 577 static int
 578 mmap_split(PFILE_MAPPING_TRACKER mt, void *begin, void *end)
 579 {
 580         LOG(4, "begin %p end %p", begin, end);
 581
 582         ASSERTeq((uintptr_t)begin % Mmap_align, 0);
 583         ASSERTeq((uintptr_t)end % Mmap_align, 0);
 584
 585         PFILE_MAPPING_TRACKER mtb = NULL;
 586         PFILE_MAPPING_TRACKER mte = NULL;
 587         HANDLE fh = mt->FileHandle;
 588         HANDLE fmh = mt->FileMappingHandle;
 589         size_t len;
 590
 591         /*
 592          * In this routine we copy flags from mt to the two subsets that we
 593          * create.  All flags may not be appropriate to propagate so let's
 594          * assert about the flags we know, if some one adds a new flag in the
 595          * future they would know about this copy and take appropricate action.
 596          */
 597         C_ASSERT(FILE_MAPPING_TRACKER_FLAGS_MASK == 1);
 598
 599         /*
 600          * 1)    b    e           b     e
 601          *    xxxxxxxxxxxxx => xxx.......xxxx  -  mtb+mte
 602          * 2)       b     e           b     e
 603          *    xxxxxxxxxxxxx => xxxxxxx.......  -  mtb
 604          * 3) b     e          b      e
 605          *    xxxxxxxxxxxxx => ........xxxxxx  -  mte
 606          * 4) b           e    b            e
 607          *    xxxxxxxxxxxxx => ..............  -  <none>
 608          */
 609
 610         if (begin > mt->BaseAddress) {
 611                 /* case #1/2 */
 612                 /* new mapping at the beginning */
 613                 mtb = malloc(sizeof(struct FILE_MAPPING_TRACKER));
 614                 if (mtb == NULL) {
 615                         ERR("!malloc");
 616                         goto err;
 617                 }
 618
 619                 mtb->Flags = mt->Flags;
 620                 mtb->FileHandle = fh;
 621                 mtb->FileMappingHandle = fmh;
 622                 mtb->BaseAddress = mt->BaseAddress;
 623                 mtb->EndAddress = begin;
 624                 mtb->Access = mt->Access;
 625                 mtb->Offset = mt->Offset;
 626
 627                 len = (char *)begin - (char *)mt->BaseAddress;
 628                 mtb->FileLen = len >= mt->FileLen ? mt->FileLen : len;
 629         }
 630
 631         if (end < mt->EndAddress) {
 632                 /* case #1/3 */
 633                 /* new mapping at the end */
 634                 mte = malloc(sizeof(struct FILE_MAPPING_TRACKER));
 635                 if (mte == NULL) {
 636                         ERR("!malloc");
 637                         goto err;
 638                 }
 639
 640                 if (!mtb) {
 641                         /* case #3 */
 642                         mte->FileHandle = fh;
 643                         mte->FileMappingHandle = fmh;
 644                 } else {
 645                         /* case #1 - need to duplicate handles */
 646                         mte->FileHandle = NULL;
 647                         mte->FileMappingHandle = NULL;
 648
 649                         if (!DuplicateHandle(GetCurrentProcess(), fh,
 650                                         GetCurrentProcess(),
 651                                         &mte->FileHandle,
 652                                         0, FALSE, DUPLICATE_SAME_ACCESS)) {
 653                                 ERR("DuplicateHandle, gle: 0x%08x",
 654                                         GetLastError());
 655                                 goto err;
 656                         }
 657
 658                         if (!DuplicateHandle(GetCurrentProcess(), fmh,
 659                                         GetCurrentProcess(),
 660                                         &mte->FileMappingHandle,
 661                                         0, FALSE, DUPLICATE_SAME_ACCESS)) {
 662                                 ERR("DuplicateHandle, gle: 0x%08x",
 663                                         GetLastError());
 664                                 goto err;
 665                         }
 666                 }
 667
 668                 mte->Flags = mt->Flags;
 669                 mte->BaseAddress = end;
 670                 mte->EndAddress = mt->EndAddress;
 671                 mte->Access = mt->Access;
 672                 mte->Offset = mt->Offset +
 673                         ((char *)mte->BaseAddress - (char *)mt->BaseAddress);
 674
 675                 len = (char *)end - (char *)mt->BaseAddress;
 676                 mte->FileLen = len >= mt->FileLen ? 0 : mt->FileLen - len;
 677         }
 678
 679         if (mt->FileLen > 0 && UnmapViewOfFile(mt->BaseAddress) == FALSE) {
 680                 ERR("UnmapViewOfFile, gle: 0x%08x", GetLastError());
 681                 goto err;
 682         }
 683
 684         len = (char *)mt->EndAddress - (char *)mt->BaseAddress;
 685         if (len > mt->FileLen) {
 686                 void *addr = (char *)mt->BaseAddress + mt->FileLen;
 687                 mmap_unreserve(addr, len - mt->FileLen);
 688         }
 689
 690         if (!mtb && !mte) {
 691                 /* case #4 */
 692                 CloseHandle(fmh);
 693                 CloseHandle(fh);
 694         }
 695
 696         /*
 697          * free entry for the original mapping
 698          */
 699         PMDK_SORTEDQ_REMOVE(&FileMappingQHead, mt, ListEntry);
 700         free(mt);
 701
 702         if (mtb) {
 703                 len = (char *)mtb->EndAddress - (char *)mtb->BaseAddress;
 704                 if (len > mtb->FileLen) {
 705                         void *addr = (char *)mtb->BaseAddress + mtb->FileLen;
 706                         void *raddr = mmap_reserve(addr, len - mtb->FileLen);
 707                         if (raddr == MAP_FAILED) {
 708                                 ERR("cannot find a contiguous region - "
 709                                         "addr: %p, len: %lx, gle: 0x%08x",
 710                                         addr, len, GetLastError());
 711                                 goto err;
 712                         }
 713                 }
 714
 715                 if (mtb->FileLen > 0) {
 716                         void *base = MapViewOfFileEx(mtb->FileMappingHandle,
 717                                 mtb->Access,
 718                                 (DWORD) (mtb->Offset >> 32),
 719                                 (DWORD) (mtb->Offset & 0xFFFFFFFF),
 720                                 mtb->FileLen,
 721                                 mtb->BaseAddress); /* hint address */
 722
 723                         if (base == NULL) {
 724                                 ERR("MapViewOfFileEx, gle: 0x%08x",
 725                                                 GetLastError());
 726                                 goto err;
 727                         }
 728                 }
 729
 730                 PMDK_SORTEDQ_INSERT(&FileMappingQHead, mtb, ListEntry,
 731                         FILE_MAPPING_TRACKER, mmap_file_mapping_comparer);
 732         }
 733
 734         if (mte) {
 735                 len = (char *)mte->EndAddress - (char *)mte->BaseAddress;
 736                 if (len > mte->FileLen) {
 737                         void *addr = (char *)mte->BaseAddress + mte->FileLen;
 738                         void *raddr = mmap_reserve(addr, len - mte->FileLen);
 739                         if (raddr == MAP_FAILED) {
 740                                 ERR("cannot find a contiguous region - "
 741                                         "addr: %p, len: %lx, gle: 0x%08x",
 742                                         addr, len, GetLastError());
 743                                 goto err;
 744                         }
 745                 }
 746
 747                 if (mte->FileLen > 0) {
 748                         void *base = MapViewOfFileEx(mte->FileMappingHandle,
 749                                 mte->Access,
 750                                 (DWORD) (mte->Offset >> 32),
 751                                 (DWORD) (mte->Offset & 0xFFFFFFFF),
 752                                 mte->FileLen,
 753                                 mte->BaseAddress); /* hint address */
 754
 755                         if (base == NULL) {
 756                                 ERR("MapViewOfFileEx, gle: 0x%08x",
 757                                                 GetLastError());
 758                                 goto err_mte;
 759                         }
 760                 }
 761
 762                 PMDK_SORTEDQ_INSERT(&FileMappingQHead, mte, ListEntry,
 763                         FILE_MAPPING_TRACKER, mmap_file_mapping_comparer);
 764         }
 765
 766         return 0;
 767
 768 err:
 769         if (mtb) {
 770                 ASSERTeq(mtb->FileMappingHandle, fmh);
 771                 ASSERTeq(mtb->FileHandle, fh);
 772                 CloseHandle(mtb->FileMappingHandle);
 773                 CloseHandle(mtb->FileHandle);
 774
 775                 len = (char *)mtb->EndAddress - (char *)mtb->BaseAddress;
 776                 if (len > mtb->FileLen) {
 777                         void *addr = (char *)mtb->BaseAddress + mtb->FileLen;
 778                         mmap_unreserve(addr, len - mtb->FileLen);
 779                 }
 780         }
 781
 782 err_mte:
 783         if (mte) {
 784                 if (mte->FileMappingHandle)
 785                         CloseHandle(mte->FileMappingHandle);
 786                 if (mte->FileHandle)
 787                         CloseHandle(mte->FileHandle);
 788
 789                 len = (char *)mte->EndAddress - (char *)mte->BaseAddress;
 790                 if (len > mte->FileLen) {
 791                         void *addr = (char *)mte->BaseAddress + mte->FileLen;
 792                         mmap_unreserve(addr, len - mte->FileLen);
 793                 }
 794         }
 795
 796         free(mtb);
 797         free(mte);
 798         return -1;
 799 }
 800
 801 /*
 802  * munmap -- delete mapping
 803  */
 804 int
 805 munmap(void *addr, size_t len)
 806 {
 807         LOG(4, "addr %p len %zu", addr, len);
 808
 809         if (((uintptr_t)addr % Mmap_align) != 0) {
 810                 ERR("address is not well-aligned: %p", addr);
 811                 errno = EINVAL;
 812                 return -1;
 813         }
 814
 815         if (len == 0) {
 816                 ERR("invalid length: %zu", len);
 817                 errno = EINVAL;
 818                 return -1;
 819         }
 820
 821         int retval = -1;
 822
 823         if (len > UINTPTR_MAX - (uintptr_t)addr) {
 824                 /* limit len to not get beyond address space */
 825                 len = UINTPTR_MAX - (uintptr_t)addr;
 826         }
 827
 828         void *begin = addr;
 829         void *end = (void *)((char *)addr + len);
 830
 831         AcquireSRWLockExclusive(&FileMappingQLock);
 832
 833         PFILE_MAPPING_TRACKER mt;
 834         PFILE_MAPPING_TRACKER next;
 835         for (mt = PMDK_SORTEDQ_FIRST(&FileMappingQHead);
 836                 mt != (void *)&FileMappingQHead;
 837                 mt = next) {
 838
 839                 /*
 840                  * Pick the next entry before we split there by delete the
 841                  * this one (NOTE: mmap_spilt could delete this entry).
 842                  */
 843                 next = PMDK_SORTEDQ_NEXT(mt, ListEntry);
 844
 845                 if (mt->BaseAddress >= end) {
 846                         LOG(4, "ignoring all mapped ranges beyond given range");
 847                         break;
 848                 }
 849
 850                 if (mt->EndAddress <= begin) {
 851                         LOG(4, "skipping a mapped range before given range");
 852                         continue;
 853                 }
 854
 855                 void *begin2 = begin > mt->BaseAddress ?
 856                                 begin : mt->BaseAddress;
 857                 void *end2 = end < mt->EndAddress ?
 858                                 end : mt->EndAddress;
 859
 860                 size_t len2 = (char *)end2 - (char *)begin2;
 861
 862                 void *align_end = (void *)roundup((uintptr_t)end2, Mmap_align);
 863                 if (mmap_split(mt, begin2, align_end) != 0) {
 864                         LOG(2, "mapping split failed");
 865                         goto err;
 866                 }
 867
 868                 if (len > len2) {
 869                         len -= len2;
 870                 } else {
 871                         len = 0;
 872                         break;
 873                 }
 874         }
 875
 876         /*
 877          * If we didn't find any mapped regions in our list attempt to free
 878          * as if the entire range is reserved.
 879          *
 880          * XXX: We don't handle a range having few mapped regions and few
 881          * reserved regions.
 882          */
 883         if (len > 0)
 884                 mmap_unreserve(addr, roundup(len, Mmap_align));
 885
 886         retval = 0;
 887
 888 err:
 889         ReleaseSRWLockExclusive(&FileMappingQLock);
 890
 891         if (retval == -1)
 892                 errno = EINVAL;
 893
 894 #ifdef MMAP_DEBUG_INFO
 895         mmap_info();
 896 #endif
 897
 898         return retval;
 899 }
 900
 901 #define MS_ALL (MS_SYNC|MS_ASYNC|MS_INVALIDATE)
 902
 903 /*
 904  * msync -- synchronize a file with a memory map
 905  */
 906 int
 907 msync(void *addr, size_t len, int flags)
 908 {
 909         LOG(4, "addr %p len %zu flags %d", addr, len, flags);
 910
 911         if ((flags & ~MS_ALL) != 0) {
 912                 ERR("invalid flags: 0x%08x", flags);
 913                 errno = EINVAL;
 914                 return -1;
 915         }
 916
 917         /*
 918          * XXX - On Linux it is allowed to call msync() without MS_SYNC
 919          * nor MS_ASYNC.
 920          */
 921         if (((flags & MS_SYNC) && (flags & MS_ASYNC)) ||
 922             ((flags & (MS_SYNC | MS_ASYNC)) == 0)) {
 923                 ERR("neither MS_SYNC or MS_ASYNC is set, or both: 0x%08x",
 924                         flags);
 925                 errno = EINVAL;
 926                 return -1;
 927         }
 928
 929         if (((uintptr_t)addr % Pagesize) != 0) {
 930                 ERR("address is not page-aligned: %p", addr);
 931                 errno = EINVAL;
 932                 return -1;
 933         }
 934
 935         if (len == 0) {
 936                 LOG(4, "zero-length region - do nothing");
 937                 return 0; /* do nothing */
 938         }
 939
 940         if (len > UINTPTR_MAX - (uintptr_t)addr) {
 941                 /* limit len to not get beyond address space */
 942                 len = UINTPTR_MAX - (uintptr_t)addr;
 943         }
 944
 945         int retval = -1;
 946
 947         void *begin = addr;
 948         void *end = (void *)((char *)addr + len);
 949
 950         AcquireSRWLockShared(&FileMappingQLock);
 951
 952         PFILE_MAPPING_TRACKER mt;
 953         PMDK_SORTEDQ_FOREACH(mt, &FileMappingQHead, ListEntry) {
 954                 if (mt->BaseAddress >= end) {
 955                         LOG(4, "ignoring all mapped ranges beyond given range");
 956                         break;
 957                 }
 958                 if (mt->EndAddress <= begin) {
 959                         LOG(4, "skipping a mapped range before given range");
 960                         continue;
 961                 }
 962
 963                 void *begin2 = begin > mt->BaseAddress ?
 964                         begin : mt->BaseAddress;
 965                 void *end2 = end < mt->EndAddress ?
 966                         end : mt->EndAddress;
 967
 968                 size_t len2 = (char *)end2 - (char *)begin2;
 969
 970                 /* do nothing for anonymous mappings */
 971                 if (mt->FileHandle != INVALID_HANDLE_VALUE) {
 972                         if (FlushViewOfFile(begin2, len2) == FALSE) {
 973                                 ERR("FlushViewOfFile, gle: 0x%08x",
 974                                         GetLastError());
 975                                 errno = ENOMEM;
 976                                 goto err;
 977                         }
 978
 979                         if (FlushFileBuffers(mt->FileHandle) == FALSE) {
 980                                 ERR("FlushFileBuffers, gle: 0x%08x",
 981                                         GetLastError());
 982                                 errno = EINVAL;
 983                                 goto err;
 984                         }
 985                 }
 986
 987                 if (len > len2) {
 988                         len -= len2;
 989                 } else {
 990                         len = 0;
 991                         break;
 992                 }
 993         }
 994
 995         if (len > 0) {
 996                 ERR("indicated memory (or part of it) was not mapped");
 997                 errno = ENOMEM;
 998         } else {
 999                 retval = 0;
1000         }
1001
1002 err:
1003         ReleaseSRWLockShared(&FileMappingQLock);
1004         return retval;
1005 }
1006
1007 #define PROT_ALL (PROT_READ|PROT_WRITE|PROT_EXEC)
1008
1009 /*
1010  * mprotect -- set protection on a region of memory
1011  *
1012  * XXX - If the memory range passed to mprotect() includes invalid pages,
1013  * returned status will indicate error, and errno is set to ENOMEM.
1014  * However, the protection change is actually applied to all the valid pages,
1015  * ignoring the rest.
1016  * This is different than on Linux, where it stops on the first invalid page.
1017  */
1018 int
1019 mprotect(void *addr, size_t len, int prot)
1020 {
1021         LOG(4, "addr %p len %zu prot %d", addr, len, prot);
1022
1023         if (((uintptr_t)addr % Pagesize) != 0) {
1024                 ERR("address is not page-aligned: %p", addr);
1025                 errno = EINVAL;
1026                 return -1;
1027         }
1028
1029         if (len == 0) {
1030                 LOG(4, "zero-length region - do nothing");
1031                 return 0; /* do nothing */
1032         }
1033
1034         if (len > UINTPTR_MAX - (uintptr_t)addr) {
1035                 len = UINTPTR_MAX - (uintptr_t)addr;
1036                 LOG(4, "limit len to %zu to not get beyond address space", len);
1037         }
1038
1039         DWORD protect = 0;
1040
1041         if ((prot & PROT_READ) && (prot & PROT_WRITE)) {
1042                 protect |= PAGE_READWRITE;
1043                 if (prot & PROT_EXEC)
1044                         protect |= PAGE_EXECUTE_READWRITE;
1045         } else if (prot & PROT_READ) {
1046                 protect |= PAGE_READONLY;
1047                 if (prot & PROT_EXEC)
1048                         protect |= PAGE_EXECUTE_READ;
1049         } else {
1050                 protect |= PAGE_NOACCESS;
1051         }
1052
1053         int retval = -1;
1054
1055         void *begin = addr;
1056         void *end = (void *)((char *)addr + len);
1057
1058         AcquireSRWLockShared(&FileMappingQLock);
1059
1060         PFILE_MAPPING_TRACKER mt;
1061         PMDK_SORTEDQ_FOREACH(mt, &FileMappingQHead, ListEntry) {
1062                 if (mt->BaseAddress >= end) {
1063                         LOG(4, "ignoring all mapped ranges beyond given range");
1064                         break;
1065                 }
1066                 if (mt->EndAddress <= begin) {
1067                         LOG(4, "skipping a mapped range before given range");
1068                         continue;
1069                 }
1070
1071                 void *begin2 = begin > mt->BaseAddress ?
1072                                 begin : mt->BaseAddress;
1073                 void *end2 = end < mt->EndAddress ?
1074                                 end : mt->EndAddress;
1075
1076                 /*
1077                  * protect of region to VirtualProtection must be compatible
1078                  * with the access protection specified for this region
1079                  * when the view was mapped using MapViewOfFileEx
1080                  */
1081                 if (mt->Access == FILE_MAP_COPY) {
1082                         if (protect & PAGE_READWRITE) {
1083                                 protect &= ~PAGE_READWRITE;
1084                                 protect |= PAGE_WRITECOPY;
1085                         } else if (protect & PAGE_EXECUTE_READWRITE) {
1086                                 protect &= ~PAGE_EXECUTE_READWRITE;
1087                                 protect |= PAGE_EXECUTE_WRITECOPY;
1088                         }
1089                 }
1090
1091                 size_t len2 = (char *)end2 - (char *)begin2;
1092
1093                 DWORD oldprot = 0;
1094                 BOOL ret;
1095                 ret = VirtualProtect(begin2, len2, protect, &oldprot);
1096                 if (ret == FALSE) {
1097                         DWORD gle = GetLastError();
1098                         ERR("VirtualProtect, gle: 0x%08x", gle);
1099                         /* translate error code */
1100                         switch (gle) {
1101                                 case ERROR_INVALID_PARAMETER:
1102                                         errno = EACCES;
1103                                         break;
1104                                 case ERROR_INVALID_ADDRESS:
1105                                         errno = ENOMEM;
1106                                         break;
1107                                 default:
1108                                         errno = EINVAL;
1109                                         break;
1110                         }
1111                         goto err;
1112                 }
1113
1114                 if (len > len2) {
1115                         len -= len2;
1116                 } else {
1117                         len = 0;
1118                         break;
1119                 }
1120         }
1121
1122         if (len > 0) {
1123                 ERR("indicated memory (or part of it) was not mapped");
1124                 errno = ENOMEM;
1125         } else {
1126                 retval = 0;
1127         }
1128
1129 err:
1130         ReleaseSRWLockShared(&FileMappingQLock);
1131         return retval;
1132 }