UefiCpuPkg/CpuDxe/CpuPageTable.c

   1 /** @file
   2   Page table management support.
   3
   4   Copyright (c) 2017, Intel Corporation. All rights reserved.<BR>
   5   Copyright (c) 2017, AMD Incorporated. All rights reserved.<BR>
   6
   7   This program and the accompanying materials
   8   are licensed and made available under the terms and conditions of the BSD License
   9   which accompanies this distribution.  The full text of the license may be found at
  10   http://opensource.org/licenses/bsd-license.php
  11
  12   THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  13   WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  14
  15 **/
  16
  17 #include <Base.h>
  18 #include <Uefi.h>
  19 #include <Library/BaseLib.h>
  20 #include <Library/CpuLib.h>
  21 #include <Library/BaseMemoryLib.h>
  22 #include <Library/MemoryAllocationLib.h>
  23 #include <Library/DebugLib.h>
  24 #include <Library/UefiBootServicesTableLib.h>
  25 #include <Protocol/MpService.h>
  26 #include <Protocol/SmmBase2.h>
  27
  28 #include "CpuDxe.h"
  29 #include "CpuPageTable.h"
  30
  31 ///
  32 /// Page Table Entry
  33 ///
  34 #define IA32_PG_P                   BIT0
  35 #define IA32_PG_RW                  BIT1
  36 #define IA32_PG_U                   BIT2
  37 #define IA32_PG_WT                  BIT3
  38 #define IA32_PG_CD                  BIT4
  39 #define IA32_PG_A                   BIT5
  40 #define IA32_PG_D                   BIT6
  41 #define IA32_PG_PS                  BIT7
  42 #define IA32_PG_PAT_2M              BIT12
  43 #define IA32_PG_PAT_4K              IA32_PG_PS
  44 #define IA32_PG_PMNT                BIT62
  45 #define IA32_PG_NX                  BIT63
  46
  47 #define PAGE_ATTRIBUTE_BITS         (IA32_PG_D | IA32_PG_A | IA32_PG_U | IA32_PG_RW | IA32_PG_P)
  48 //
  49 // Bits 1, 2, 5, 6 are reserved in the IA32 PAE PDPTE
  50 // X64 PAE PDPTE does not have such restriction
  51 //
  52 #define IA32_PAE_PDPTE_ATTRIBUTE_BITS    (IA32_PG_P)
  53
  54 #define PAGE_PROGATE_BITS           (IA32_PG_NX | PAGE_ATTRIBUTE_BITS)
  55
  56 #define PAGING_4K_MASK  0xFFF
  57 #define PAGING_2M_MASK  0x1FFFFF
  58 #define PAGING_1G_MASK  0x3FFFFFFF
  59
  60 #define PAGING_PAE_INDEX_MASK  0x1FF
  61
  62 #define PAGING_4K_ADDRESS_MASK_64 0x000FFFFFFFFFF000ull
  63 #define PAGING_2M_ADDRESS_MASK_64 0x000FFFFFFFE00000ull
  64 #define PAGING_1G_ADDRESS_MASK_64 0x000FFFFFC0000000ull
  65
  66 typedef enum {
  67   PageNone,
  68   Page4K,
  69   Page2M,
  70   Page1G,
  71 } PAGE_ATTRIBUTE;
  72
  73 typedef struct {
  74   PAGE_ATTRIBUTE   Attribute;
  75   UINT64           Length;
  76   UINT64           AddressMask;
  77 } PAGE_ATTRIBUTE_TABLE;
  78
  79 typedef enum {
  80   PageActionAssign,
  81   PageActionSet,
  82   PageActionClear,
  83 } PAGE_ACTION;
  84
  85 PAGE_ATTRIBUTE_TABLE mPageAttributeTable[] = {
  86   {Page4K,  SIZE_4KB, PAGING_4K_ADDRESS_MASK_64},
  87   {Page2M,  SIZE_2MB, PAGING_2M_ADDRESS_MASK_64},
  88   {Page1G,  SIZE_1GB, PAGING_1G_ADDRESS_MASK_64},
  89 };
  90
  91 PAGE_TABLE_POOL                   *mPageTablePool = NULL;
  92 PAGE_TABLE_LIB_PAGING_CONTEXT     mPagingContext;
  93 EFI_SMM_BASE2_PROTOCOL            *mSmmBase2 = NULL;
  94
  95 /**
  96  Check if current execution environment is in SMM mode or not, via
  97  EFI_SMM_BASE2_PROTOCOL.
  98
  99  This is necessary because of the fact that MdePkg\Library\SmmMemoryAllocationLib
 100  supports to free memory outside SMRAM. The library will call gBS->FreePool() or
 101  gBS->FreePages() and then SetMemorySpaceAttributes interface in turn to change
 102  memory paging attributes during free operation, if some memory related features
 103  are enabled (like Heap Guard).
 104
 105  This means that SetMemorySpaceAttributes() has chance to run in SMM mode. This
 106  will cause incorrect result because SMM mode always loads its own page tables,
 107  which are usually different from DXE. This function can be used to detect such
 108  situation and help to avoid further misoperations.
 109
 110   @retval TRUE    In SMM mode.
 111   @retval FALSE   Not in SMM mode.
 112 **/
 113 BOOLEAN
 114 IsInSmm (
 115   VOID
 116   )
 117 {
 118   BOOLEAN                 InSmm;
 119
 120   InSmm = FALSE;
 121   if (mSmmBase2 == NULL) {
 122     gBS->LocateProtocol (&gEfiSmmBase2ProtocolGuid, NULL, (VOID **)&mSmmBase2);
 123   }
 124
 125   if (mSmmBase2 != NULL) {
 126     mSmmBase2->InSmm (mSmmBase2, &InSmm);
 127   }
 128
 129   return InSmm;
 130 }
 131
 132 /**
 133   Return current paging context.
 134
 135   @param[in,out]  PagingContext     The paging context.
 136 **/
 137 VOID
 138 GetCurrentPagingContext (
 139   IN OUT PAGE_TABLE_LIB_PAGING_CONTEXT     *PagingContext
 140   )
 141 {
 142   UINT32                         RegEax;
 143   UINT32                         RegEdx;
 144
 145   //
 146   // Don't retrieve current paging context from processor if in SMM mode.
 147   //
 148   if (!IsInSmm ()) {
 149     ZeroMem (&mPagingContext, sizeof(mPagingContext));
 150     if (sizeof(UINTN) == sizeof(UINT64)) {
 151       mPagingContext.MachineType = IMAGE_FILE_MACHINE_X64;
 152     } else {
 153       mPagingContext.MachineType = IMAGE_FILE_MACHINE_I386;
 154     }
 155     if ((AsmReadCr0 () & BIT31) != 0) {
 156       mPagingContext.ContextData.X64.PageTableBase = (AsmReadCr3 () & PAGING_4K_ADDRESS_MASK_64);
 157     } else {
 158       mPagingContext.ContextData.X64.PageTableBase = 0;
 159     }
 160
 161     if ((AsmReadCr4 () & BIT4) != 0) {
 162       mPagingContext.ContextData.Ia32.Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PSE;
 163     }
 164     if ((AsmReadCr4 () & BIT5) != 0) {
 165       mPagingContext.ContextData.Ia32.Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE;
 166     }
 167     if ((AsmReadCr0 () & BIT16) != 0) {
 168       mPagingContext.ContextData.Ia32.Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_WP_ENABLE;
 169     }
 170
 171     AsmCpuid (0x80000000, &RegEax, NULL, NULL, NULL);
 172     if (RegEax > 0x80000000) {
 173       AsmCpuid (0x80000001, NULL, NULL, NULL, &RegEdx);
 174       if ((RegEdx & BIT20) != 0) {
 175         // XD supported
 176         if ((AsmReadMsr64 (0xC0000080) & BIT11) != 0) {
 177           // XD activated
 178           mPagingContext.ContextData.Ia32.Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_XD_ACTIVATED;
 179         }
 180       }
 181       if ((RegEdx & BIT26) != 0) {
 182         mPagingContext.ContextData.Ia32.Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAGE_1G_SUPPORT;
 183       }
 184     }
 185   }
 186
 187   //
 188   // This can avoid getting SMM paging context if in SMM mode. We cannot assume
 189   // SMM mode shares the same paging context as DXE.
 190   //
 191   CopyMem (PagingContext, &mPagingContext, sizeof (mPagingContext));
 192 }
 193
 194 /**
 195   Return length according to page attributes.
 196
 197   @param[in]  PageAttributes   The page attribute of the page entry.
 198
 199   @return The length of page entry.
 200 **/
 201 UINTN
 202 PageAttributeToLength (
 203   IN PAGE_ATTRIBUTE  PageAttribute
 204   )
 205 {
 206   UINTN  Index;
 207   for (Index = 0; Index < sizeof(mPageAttributeTable)/sizeof(mPageAttributeTable[0]); Index++) {
 208     if (PageAttribute == mPageAttributeTable[Index].Attribute) {
 209       return (UINTN)mPageAttributeTable[Index].Length;
 210     }
 211   }
 212   return 0;
 213 }
 214
 215 /**
 216   Return address mask according to page attributes.
 217
 218   @param[in]  PageAttributes   The page attribute of the page entry.
 219
 220   @return The address mask of page entry.
 221 **/
 222 UINTN
 223 PageAttributeToMask (
 224   IN PAGE_ATTRIBUTE  PageAttribute
 225   )
 226 {
 227   UINTN  Index;
 228   for (Index = 0; Index < sizeof(mPageAttributeTable)/sizeof(mPageAttributeTable[0]); Index++) {
 229     if (PageAttribute == mPageAttributeTable[Index].Attribute) {
 230       return (UINTN)mPageAttributeTable[Index].AddressMask;
 231     }
 232   }
 233   return 0;
 234 }
 235
 236 /**
 237   Return page table entry to match the address.
 238
 239   @param[in]  PagingContext     The paging context.
 240   @param[in]  Address           The address to be checked.
 241   @param[out] PageAttributes    The page attribute of the page entry.
 242
 243   @return The page entry.
 244 **/
 245 VOID *
 246 GetPageTableEntry (
 247   IN  PAGE_TABLE_LIB_PAGING_CONTEXT     *PagingContext,
 248   IN  PHYSICAL_ADDRESS                  Address,
 249   OUT PAGE_ATTRIBUTE                    *PageAttribute
 250   )
 251 {
 252   UINTN                 Index1;
 253   UINTN                 Index2;
 254   UINTN                 Index3;
 255   UINTN                 Index4;
 256   UINT64                *L1PageTable;
 257   UINT64                *L2PageTable;
 258   UINT64                *L3PageTable;
 259   UINT64                *L4PageTable;
 260   UINT64                AddressEncMask;
 261
 262   ASSERT (PagingContext != NULL);
 263
 264   Index4 = ((UINTN)RShiftU64 (Address, 39)) & PAGING_PAE_INDEX_MASK;
 265   Index3 = ((UINTN)Address >> 30) & PAGING_PAE_INDEX_MASK;
 266   Index2 = ((UINTN)Address >> 21) & PAGING_PAE_INDEX_MASK;
 267   Index1 = ((UINTN)Address >> 12) & PAGING_PAE_INDEX_MASK;
 268
 269   // Make sure AddressEncMask is contained to smallest supported address field.
 270   //
 271   AddressEncMask = PcdGet64 (PcdPteMemoryEncryptionAddressOrMask) & PAGING_1G_ADDRESS_MASK_64;
 272
 273   if (PagingContext->MachineType == IMAGE_FILE_MACHINE_X64) {
 274     L4PageTable = (UINT64 *)(UINTN)PagingContext->ContextData.X64.PageTableBase;
 275     if (L4PageTable[Index4] == 0) {
 276       *PageAttribute = PageNone;
 277       return NULL;
 278     }
 279
 280     L3PageTable = (UINT64 *)(UINTN)(L4PageTable[Index4] & ~AddressEncMask & PAGING_4K_ADDRESS_MASK_64);
 281   } else {
 282     ASSERT((PagingContext->ContextData.Ia32.Attributes & PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE) != 0);
 283     L3PageTable = (UINT64 *)(UINTN)PagingContext->ContextData.Ia32.PageTableBase;
 284   }
 285   if (L3PageTable[Index3] == 0) {
 286     *PageAttribute = PageNone;
 287     return NULL;
 288   }
 289   if ((L3PageTable[Index3] & IA32_PG_PS) != 0) {
 290     // 1G
 291     *PageAttribute = Page1G;
 292     return &L3PageTable[Index3];
 293   }
 294
 295   L2PageTable = (UINT64 *)(UINTN)(L3PageTable[Index3] & ~AddressEncMask & PAGING_4K_ADDRESS_MASK_64);
 296   if (L2PageTable[Index2] == 0) {
 297     *PageAttribute = PageNone;
 298     return NULL;
 299   }
 300   if ((L2PageTable[Index2] & IA32_PG_PS) != 0) {
 301     // 2M
 302     *PageAttribute = Page2M;
 303     return &L2PageTable[Index2];
 304   }
 305
 306   // 4k
 307   L1PageTable = (UINT64 *)(UINTN)(L2PageTable[Index2] & ~AddressEncMask & PAGING_4K_ADDRESS_MASK_64);
 308   if ((L1PageTable[Index1] == 0) && (Address != 0)) {
 309     *PageAttribute = PageNone;
 310     return NULL;
 311   }
 312   *PageAttribute = Page4K;
 313   return &L1PageTable[Index1];
 314 }
 315
 316 /**
 317   Return memory attributes of page entry.
 318
 319   @param[in]  PageEntry        The page entry.
 320
 321   @return Memory attributes of page entry.
 322 **/
 323 UINT64
 324 GetAttributesFromPageEntry (
 325   IN  UINT64                            *PageEntry
 326   )
 327 {
 328   UINT64  Attributes;
 329   Attributes = 0;
 330   if ((*PageEntry & IA32_PG_P) == 0) {
 331     Attributes |= EFI_MEMORY_RP;
 332   }
 333   if ((*PageEntry & IA32_PG_RW) == 0) {
 334     Attributes |= EFI_MEMORY_RO;
 335   }
 336   if ((*PageEntry & IA32_PG_NX) != 0) {
 337     Attributes |= EFI_MEMORY_XP;
 338   }
 339   return Attributes;
 340 }
 341
 342 /**
 343   Modify memory attributes of page entry.
 344
 345   @param[in]  PagingContext    The paging context.
 346   @param[in]  PageEntry        The page entry.
 347   @param[in]  Attributes       The bit mask of attributes to modify for the memory region.
 348   @param[in]  PageAction       The page action.
 349   @param[out] IsModified       TRUE means page table modified. FALSE means page table not modified.
 350 **/
 351 VOID
 352 ConvertPageEntryAttribute (
 353   IN  PAGE_TABLE_LIB_PAGING_CONTEXT     *PagingContext,
 354   IN  UINT64                            *PageEntry,
 355   IN  UINT64                            Attributes,
 356   IN  PAGE_ACTION                       PageAction,
 357   OUT BOOLEAN                           *IsModified
 358   )
 359 {
 360   UINT64  CurrentPageEntry;
 361   UINT64  NewPageEntry;
 362
 363   CurrentPageEntry = *PageEntry;
 364   NewPageEntry = CurrentPageEntry;
 365   if ((Attributes & EFI_MEMORY_RP) != 0) {
 366     switch (PageAction) {
 367     case PageActionAssign:
 368     case PageActionSet:
 369       NewPageEntry &= ~(UINT64)IA32_PG_P;
 370       break;
 371     case PageActionClear:
 372       NewPageEntry |= IA32_PG_P;
 373       break;
 374     }
 375   } else {
 376     switch (PageAction) {
 377     case PageActionAssign:
 378       NewPageEntry |= IA32_PG_P;
 379       break;
 380     case PageActionSet:
 381     case PageActionClear:
 382       break;
 383     }
 384   }
 385   if ((Attributes & EFI_MEMORY_RO) != 0) {
 386     switch (PageAction) {
 387     case PageActionAssign:
 388     case PageActionSet:
 389       NewPageEntry &= ~(UINT64)IA32_PG_RW;
 390       break;
 391     case PageActionClear:
 392       NewPageEntry |= IA32_PG_RW;
 393       break;
 394     }
 395   } else {
 396     switch (PageAction) {
 397     case PageActionAssign:
 398       NewPageEntry |= IA32_PG_RW;
 399       break;
 400     case PageActionSet:
 401     case PageActionClear:
 402       break;
 403     }
 404   }
 405   if ((PagingContext->ContextData.Ia32.Attributes & PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_XD_ACTIVATED) != 0) {
 406     if ((Attributes & EFI_MEMORY_XP) != 0) {
 407       switch (PageAction) {
 408       case PageActionAssign:
 409       case PageActionSet:
 410         NewPageEntry |= IA32_PG_NX;
 411         break;
 412       case PageActionClear:
 413         NewPageEntry &= ~IA32_PG_NX;
 414         break;
 415       }
 416     } else {
 417       switch (PageAction) {
 418       case PageActionAssign:
 419         NewPageEntry &= ~IA32_PG_NX;
 420         break;
 421       case PageActionSet:
 422       case PageActionClear:
 423         break;
 424       }
 425     }
 426   }
 427   *PageEntry = NewPageEntry;
 428   if (CurrentPageEntry != NewPageEntry) {
 429     *IsModified = TRUE;
 430     DEBUG ((DEBUG_VERBOSE, "ConvertPageEntryAttribute 0x%lx", CurrentPageEntry));
 431     DEBUG ((DEBUG_VERBOSE, "->0x%lx\n", NewPageEntry));
 432   } else {
 433     *IsModified = FALSE;
 434   }
 435 }
 436
 437 /**
 438   This function returns if there is need to split page entry.
 439
 440   @param[in]  BaseAddress      The base address to be checked.
 441   @param[in]  Length           The length to be checked.
 442   @param[in]  PageEntry        The page entry to be checked.
 443   @param[in]  PageAttribute    The page attribute of the page entry.
 444
 445   @retval SplitAttributes on if there is need to split page entry.
 446 **/
 447 PAGE_ATTRIBUTE
 448 NeedSplitPage (
 449   IN  PHYSICAL_ADDRESS                  BaseAddress,
 450   IN  UINT64                            Length,
 451   IN  UINT64                            *PageEntry,
 452   IN  PAGE_ATTRIBUTE                    PageAttribute
 453   )
 454 {
 455   UINT64                PageEntryLength;
 456
 457   PageEntryLength = PageAttributeToLength (PageAttribute);
 458
 459   if (((BaseAddress & (PageEntryLength - 1)) == 0) && (Length >= PageEntryLength)) {
 460     return PageNone;
 461   }
 462
 463   if (((BaseAddress & PAGING_2M_MASK) != 0) || (Length < SIZE_2MB)) {
 464     return Page4K;
 465   }
 466
 467   return Page2M;
 468 }
 469
 470 /**
 471   This function splits one page entry to small page entries.
 472
 473   @param[in]  PageEntry         The page entry to be splitted.
 474   @param[in]  PageAttribute     The page attribute of the page entry.
 475   @param[in]  SplitAttribute    How to split the page entry.
 476   @param[in]  AllocatePagesFunc If page split is needed, this function is used to allocate more pages.
 477
 478   @retval RETURN_SUCCESS            The page entry is splitted.
 479   @retval RETURN_UNSUPPORTED        The page entry does not support to be splitted.
 480   @retval RETURN_OUT_OF_RESOURCES   No resource to split page entry.
 481 **/
 482 RETURN_STATUS
 483 SplitPage (
 484   IN  UINT64                            *PageEntry,
 485   IN  PAGE_ATTRIBUTE                    PageAttribute,
 486   IN  PAGE_ATTRIBUTE                    SplitAttribute,
 487   IN  PAGE_TABLE_LIB_ALLOCATE_PAGES     AllocatePagesFunc
 488   )
 489 {
 490   UINT64   BaseAddress;
 491   UINT64   *NewPageEntry;
 492   UINTN    Index;
 493   UINT64   AddressEncMask;
 494
 495   ASSERT (PageAttribute == Page2M || PageAttribute == Page1G);
 496
 497   ASSERT (AllocatePagesFunc != NULL);
 498
 499   // Make sure AddressEncMask is contained to smallest supported address field.
 500   //
 501   AddressEncMask = PcdGet64 (PcdPteMemoryEncryptionAddressOrMask) & PAGING_1G_ADDRESS_MASK_64;
 502
 503   if (PageAttribute == Page2M) {
 504     //
 505     // Split 2M to 4K
 506     //
 507     ASSERT (SplitAttribute == Page4K);
 508     if (SplitAttribute == Page4K) {
 509       NewPageEntry = AllocatePagesFunc (1);
 510       DEBUG ((DEBUG_INFO, "Split - 0x%x\n", NewPageEntry));
 511       if (NewPageEntry == NULL) {
 512         return RETURN_OUT_OF_RESOURCES;
 513       }
 514       BaseAddress = *PageEntry & ~AddressEncMask & PAGING_2M_ADDRESS_MASK_64;
 515       for (Index = 0; Index < SIZE_4KB / sizeof(UINT64); Index++) {
 516         NewPageEntry[Index] = (BaseAddress + SIZE_4KB * Index) | AddressEncMask | ((*PageEntry) & PAGE_PROGATE_BITS);
 517       }
 518       (*PageEntry) = (UINT64)(UINTN)NewPageEntry | AddressEncMask | ((*PageEntry) & PAGE_ATTRIBUTE_BITS);
 519       return RETURN_SUCCESS;
 520     } else {
 521       return RETURN_UNSUPPORTED;
 522     }
 523   } else if (PageAttribute == Page1G) {
 524     //
 525     // Split 1G to 2M
 526     // No need support 1G->4K directly, we should use 1G->2M, then 2M->4K to get more compact page table.
 527     //
 528     ASSERT (SplitAttribute == Page2M || SplitAttribute == Page4K);
 529     if ((SplitAttribute == Page2M || SplitAttribute == Page4K)) {
 530       NewPageEntry = AllocatePagesFunc (1);
 531       DEBUG ((DEBUG_INFO, "Split - 0x%x\n", NewPageEntry));
 532       if (NewPageEntry == NULL) {
 533         return RETURN_OUT_OF_RESOURCES;
 534       }
 535       BaseAddress = *PageEntry & ~AddressEncMask  & PAGING_1G_ADDRESS_MASK_64;
 536       for (Index = 0; Index < SIZE_4KB / sizeof(UINT64); Index++) {
 537         NewPageEntry[Index] = (BaseAddress + SIZE_2MB * Index) | AddressEncMask | IA32_PG_PS | ((*PageEntry) & PAGE_PROGATE_BITS);
 538       }
 539       (*PageEntry) = (UINT64)(UINTN)NewPageEntry | AddressEncMask | ((*PageEntry) & PAGE_ATTRIBUTE_BITS);
 540       return RETURN_SUCCESS;
 541     } else {
 542       return RETURN_UNSUPPORTED;
 543     }
 544   } else {
 545     return RETURN_UNSUPPORTED;
 546   }
 547 }
 548
 549 /**
 550  Check the WP status in CR0 register. This bit is used to lock or unlock write
 551  access to pages marked as read-only.
 552
 553   @retval TRUE    Write protection is enabled.
 554   @retval FALSE   Write protection is disabled.
 555 **/
 556 BOOLEAN
 557 IsReadOnlyPageWriteProtected (
 558   VOID
 559   )
 560 {
 561   //
 562   // To avoid unforseen consequences, don't touch paging settings in SMM mode
 563   // in this driver.
 564   //
 565   if (!IsInSmm ()) {
 566     return ((AsmReadCr0 () & BIT16) != 0);
 567   }
 568   return FALSE;
 569 }
 570
 571 /**
 572  Disable Write Protect on pages marked as read-only.
 573 **/
 574 VOID
 575 DisableReadOnlyPageWriteProtect (
 576   VOID
 577   )
 578 {
 579   //
 580   // To avoid unforseen consequences, don't touch paging settings in SMM mode
 581   // in this driver.
 582   //
 583   if (!IsInSmm ()) {
 584     AsmWriteCr0 (AsmReadCr0 () & ~BIT16);
 585   }
 586 }
 587
 588 /**
 589  Enable Write Protect on pages marked as read-only.
 590 **/
 591 VOID
 592 EnableReadOnlyPageWriteProtect (
 593   VOID
 594   )
 595 {
 596   //
 597   // To avoid unforseen consequences, don't touch paging settings in SMM mode
 598   // in this driver.
 599   //
 600   if (!IsInSmm ()) {
 601     AsmWriteCr0 (AsmReadCr0 () | BIT16);
 602   }
 603 }
 604
 605 /**
 606   This function modifies the page attributes for the memory region specified by BaseAddress and
 607   Length from their current attributes to the attributes specified by Attributes.
 608
 609   Caller should make sure BaseAddress and Length is at page boundary.
 610
 611   @param[in]  PagingContext     The paging context. NULL means get page table from current CPU context.
 612   @param[in]  BaseAddress       The physical address that is the start address of a memory region.
 613   @param[in]  Length            The size in bytes of the memory region.
 614   @param[in]  Attributes        The bit mask of attributes to modify for the memory region.
 615   @param[in]  PageAction        The page action.
 616   @param[in]  AllocatePagesFunc If page split is needed, this function is used to allocate more pages.
 617                                 NULL mean page split is unsupported.
 618   @param[out] IsSplitted        TRUE means page table splitted. FALSE means page table not splitted.
 619   @param[out] IsModified        TRUE means page table modified. FALSE means page table not modified.
 620
 621   @retval RETURN_SUCCESS           The attributes were modified for the memory region.
 622   @retval RETURN_ACCESS_DENIED     The attributes for the memory resource range specified by
 623                                    BaseAddress and Length cannot be modified.
 624   @retval RETURN_INVALID_PARAMETER Length is zero.
 625                                    Attributes specified an illegal combination of attributes that
 626                                    cannot be set together.
 627   @retval RETURN_OUT_OF_RESOURCES  There are not enough system resources to modify the attributes of
 628                                    the memory resource range.
 629   @retval RETURN_UNSUPPORTED       The processor does not support one or more bytes of the memory
 630                                    resource range specified by BaseAddress and Length.
 631                                    The bit mask of attributes is not support for the memory resource
 632                                    range specified by BaseAddress and Length.
 633 **/
 634 RETURN_STATUS
 635 ConvertMemoryPageAttributes (
 636   IN  PAGE_TABLE_LIB_PAGING_CONTEXT     *PagingContext OPTIONAL,
 637   IN  PHYSICAL_ADDRESS                  BaseAddress,
 638   IN  UINT64                            Length,
 639   IN  UINT64                            Attributes,
 640   IN  PAGE_ACTION                       PageAction,
 641   IN  PAGE_TABLE_LIB_ALLOCATE_PAGES     AllocatePagesFunc OPTIONAL,
 642   OUT BOOLEAN                           *IsSplitted,  OPTIONAL
 643   OUT BOOLEAN                           *IsModified   OPTIONAL
 644   )
 645 {
 646   PAGE_TABLE_LIB_PAGING_CONTEXT     CurrentPagingContext;
 647   UINT64                            *PageEntry;
 648   PAGE_ATTRIBUTE                    PageAttribute;
 649   UINTN                             PageEntryLength;
 650   PAGE_ATTRIBUTE                    SplitAttribute;
 651   RETURN_STATUS                     Status;
 652   BOOLEAN                           IsEntryModified;
 653   BOOLEAN                           IsWpEnabled;
 654
 655   if ((BaseAddress & (SIZE_4KB - 1)) != 0) {
 656     DEBUG ((DEBUG_ERROR, "BaseAddress(0x%lx) is not aligned!\n", BaseAddress));
 657     return EFI_UNSUPPORTED;
 658   }
 659   if ((Length & (SIZE_4KB - 1)) != 0) {
 660     DEBUG ((DEBUG_ERROR, "Length(0x%lx) is not aligned!\n", Length));
 661     return EFI_UNSUPPORTED;
 662   }
 663   if (Length == 0) {
 664     DEBUG ((DEBUG_ERROR, "Length is 0!\n"));
 665     return RETURN_INVALID_PARAMETER;
 666   }
 667
 668   if ((Attributes & ~(EFI_MEMORY_RP | EFI_MEMORY_RO | EFI_MEMORY_XP)) != 0) {
 669     DEBUG ((DEBUG_ERROR, "Attributes(0x%lx) has unsupported bit\n", Attributes));
 670     return EFI_UNSUPPORTED;
 671   }
 672
 673   if (PagingContext == NULL) {
 674     GetCurrentPagingContext (&CurrentPagingContext);
 675   } else {
 676     CopyMem (&CurrentPagingContext, PagingContext, sizeof(CurrentPagingContext));
 677   }
 678   switch(CurrentPagingContext.MachineType) {
 679   case IMAGE_FILE_MACHINE_I386:
 680     if (CurrentPagingContext.ContextData.Ia32.PageTableBase == 0) {
 681       if (Attributes == 0) {
 682         return EFI_SUCCESS;
 683       } else {
 684         DEBUG ((DEBUG_ERROR, "PageTable is 0!\n"));
 685         return EFI_UNSUPPORTED;
 686       }
 687     }
 688     if ((CurrentPagingContext.ContextData.Ia32.Attributes & PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE) == 0) {
 689       DEBUG ((DEBUG_ERROR, "Non-PAE Paging!\n"));
 690       return EFI_UNSUPPORTED;
 691     }
 692     if ((BaseAddress + Length) > BASE_4GB) {
 693       DEBUG ((DEBUG_ERROR, "Beyond 4GB memory in 32-bit mode!\n"));
 694       return EFI_UNSUPPORTED;
 695     }
 696     break;
 697   case IMAGE_FILE_MACHINE_X64:
 698     ASSERT (CurrentPagingContext.ContextData.X64.PageTableBase != 0);
 699     break;
 700   default:
 701     ASSERT(FALSE);
 702     return EFI_UNSUPPORTED;
 703     break;
 704   }
 705
 706 //  DEBUG ((DEBUG_ERROR, "ConvertMemoryPageAttributes(%x) - %016lx, %016lx, %02lx\n", IsSet, BaseAddress, Length, Attributes));
 707
 708   if (IsSplitted != NULL) {
 709     *IsSplitted = FALSE;
 710   }
 711   if (IsModified != NULL) {
 712     *IsModified = FALSE;
 713   }
 714   if (AllocatePagesFunc == NULL) {
 715     AllocatePagesFunc = AllocatePageTableMemory;
 716   }
 717
 718   //
 719   // Make sure that the page table is changeable.
 720   //
 721   IsWpEnabled = IsReadOnlyPageWriteProtected ();
 722   if (IsWpEnabled) {
 723     DisableReadOnlyPageWriteProtect ();
 724   }
 725
 726   //
 727   // Below logic is to check 2M/4K page to make sure we donot waist memory.
 728   //
 729   Status = EFI_SUCCESS;
 730   while (Length != 0) {
 731     PageEntry = GetPageTableEntry (&CurrentPagingContext, BaseAddress, &PageAttribute);
 732     if (PageEntry == NULL) {
 733       Status = RETURN_UNSUPPORTED;
 734       goto Done;
 735     }
 736     PageEntryLength = PageAttributeToLength (PageAttribute);
 737     SplitAttribute = NeedSplitPage (BaseAddress, Length, PageEntry, PageAttribute);
 738     if (SplitAttribute == PageNone) {
 739       ConvertPageEntryAttribute (&CurrentPagingContext, PageEntry, Attributes, PageAction, &IsEntryModified);
 740       if (IsEntryModified) {
 741         if (IsModified != NULL) {
 742           *IsModified = TRUE;
 743         }
 744       }
 745       //
 746       // Convert success, move to next
 747       //
 748       BaseAddress += PageEntryLength;
 749       Length -= PageEntryLength;
 750     } else {
 751       if (AllocatePagesFunc == NULL) {
 752         Status = RETURN_UNSUPPORTED;
 753         goto Done;
 754       }
 755       Status = SplitPage (PageEntry, PageAttribute, SplitAttribute, AllocatePagesFunc);
 756       if (RETURN_ERROR (Status)) {
 757         Status = RETURN_UNSUPPORTED;
 758         goto Done;
 759       }
 760       if (IsSplitted != NULL) {
 761         *IsSplitted = TRUE;
 762       }
 763       if (IsModified != NULL) {
 764         *IsModified = TRUE;
 765       }
 766       //
 767       // Just split current page
 768       // Convert success in next around
 769       //
 770     }
 771   }
 772
 773 Done:
 774   //
 775   // Restore page table write protection, if any.
 776   //
 777   if (IsWpEnabled) {
 778     EnableReadOnlyPageWriteProtect ();
 779   }
 780   return Status;
 781 }
 782
 783 /**
 784   This function assigns the page attributes for the memory region specified by BaseAddress and
 785   Length from their current attributes to the attributes specified by Attributes.
 786
 787   Caller should make sure BaseAddress and Length is at page boundary.
 788
 789   Caller need guarentee the TPL <= TPL_NOTIFY, if there is split page request.
 790
 791   @param[in]  PagingContext     The paging context. NULL means get page table from current CPU context.
 792   @param[in]  BaseAddress       The physical address that is the start address of a memory region.
 793   @param[in]  Length            The size in bytes of the memory region.
 794   @param[in]  Attributes        The bit mask of attributes to set for the memory region.
 795   @param[in]  AllocatePagesFunc If page split is needed, this function is used to allocate more pages.
 796                                 NULL mean page split is unsupported.
 797
 798   @retval RETURN_SUCCESS           The attributes were cleared for the memory region.
 799   @retval RETURN_ACCESS_DENIED     The attributes for the memory resource range specified by
 800                                    BaseAddress and Length cannot be modified.
 801   @retval RETURN_INVALID_PARAMETER Length is zero.
 802                                    Attributes specified an illegal combination of attributes that
 803                                    cannot be set together.
 804   @retval RETURN_OUT_OF_RESOURCES  There are not enough system resources to modify the attributes of
 805                                    the memory resource range.
 806   @retval RETURN_UNSUPPORTED       The processor does not support one or more bytes of the memory
 807                                    resource range specified by BaseAddress and Length.
 808                                    The bit mask of attributes is not support for the memory resource
 809                                    range specified by BaseAddress and Length.
 810 **/
 811 RETURN_STATUS
 812 EFIAPI
 813 AssignMemoryPageAttributes (
 814   IN  PAGE_TABLE_LIB_PAGING_CONTEXT     *PagingContext OPTIONAL,
 815   IN  PHYSICAL_ADDRESS                  BaseAddress,
 816   IN  UINT64                            Length,
 817   IN  UINT64                            Attributes,
 818   IN  PAGE_TABLE_LIB_ALLOCATE_PAGES     AllocatePagesFunc OPTIONAL
 819   )
 820 {
 821   RETURN_STATUS  Status;
 822   BOOLEAN        IsModified;
 823   BOOLEAN        IsSplitted;
 824
 825 //  DEBUG((DEBUG_INFO, "AssignMemoryPageAttributes: 0x%lx - 0x%lx (0x%lx)\n", BaseAddress, Length, Attributes));
 826   Status = ConvertMemoryPageAttributes (PagingContext, BaseAddress, Length, Attributes, PageActionAssign, AllocatePagesFunc, &IsSplitted, &IsModified);
 827   if (!EFI_ERROR(Status)) {
 828     if ((PagingContext == NULL) && IsModified) {
 829       //
 830       // Flush TLB as last step.
 831       //
 832       // Note: Since APs will always init CR3 register in HLT loop mode or do
 833       // TLB flush in MWAIT loop mode, there's no need to flush TLB for them
 834       // here.
 835       //
 836       CpuFlushTlb();
 837     }
 838   }
 839
 840   return Status;
 841 }
 842
 843 /**
 844  Check if Execute Disable feature is enabled or not.
 845 **/
 846 BOOLEAN
 847 IsExecuteDisableEnabled (
 848   VOID
 849   )
 850 {
 851   MSR_CORE_IA32_EFER_REGISTER    MsrEfer;
 852
 853   MsrEfer.Uint64 = AsmReadMsr64 (MSR_IA32_EFER);
 854   return (MsrEfer.Bits.NXE == 1);
 855 }
 856
 857 /**
 858   Update GCD memory space attributes according to current page table setup.
 859 **/
 860 VOID
 861 RefreshGcdMemoryAttributesFromPaging (
 862   VOID
 863   )
 864 {
 865   EFI_STATUS                          Status;
 866   UINTN                               NumberOfDescriptors;
 867   EFI_GCD_MEMORY_SPACE_DESCRIPTOR     *MemorySpaceMap;
 868   PAGE_TABLE_LIB_PAGING_CONTEXT       PagingContext;
 869   PAGE_ATTRIBUTE                      PageAttribute;
 870   UINT64                              *PageEntry;
 871   UINT64                              PageLength;
 872   UINT64                              MemorySpaceLength;
 873   UINT64                              Length;
 874   UINT64                              BaseAddress;
 875   UINT64                              PageStartAddress;
 876   UINT64                              Attributes;
 877   UINT64                              Capabilities;
 878   UINT64                              NewAttributes;
 879   UINTN                               Index;
 880
 881   //
 882   // Assuming that memory space map returned is sorted already; otherwise sort
 883   // them in the order of lowest address to highest address.
 884   //
 885   Status = gDS->GetMemorySpaceMap (&NumberOfDescriptors, &MemorySpaceMap);
 886   ASSERT_EFI_ERROR (Status);
 887
 888   GetCurrentPagingContext (&PagingContext);
 889
 890   Attributes      = 0;
 891   NewAttributes   = 0;
 892   BaseAddress     = 0;
 893   PageLength      = 0;
 894
 895   if (IsExecuteDisableEnabled ()) {
 896     Capabilities = EFI_MEMORY_RO | EFI_MEMORY_RP | EFI_MEMORY_XP;
 897   } else {
 898     Capabilities = EFI_MEMORY_RO | EFI_MEMORY_RP;
 899   }
 900
 901   for (Index = 0; Index < NumberOfDescriptors; Index++) {
 902     if (MemorySpaceMap[Index].GcdMemoryType == EfiGcdMemoryTypeNonExistent) {
 903       continue;
 904     }
 905
 906     //
 907     // Sync the actual paging related capabilities back to GCD service first.
 908     // As a side effect (good one), this can also help to avoid unnecessary
 909     // memory map entries due to the different capabilities of the same type
 910     // memory, such as multiple RT_CODE and RT_DATA entries in memory map,
 911     // which could cause boot failure of some old Linux distro (before v4.3).
 912     //
 913     Status = gDS->SetMemorySpaceCapabilities (
 914                     MemorySpaceMap[Index].BaseAddress,
 915                     MemorySpaceMap[Index].Length,
 916                     MemorySpaceMap[Index].Capabilities | Capabilities
 917                     );
 918     if (EFI_ERROR (Status)) {
 919       //
 920       // If we cannot udpate the capabilities, we cannot update its
 921       // attributes either. So just simply skip current block of memory.
 922       //
 923       DEBUG ((
 924         DEBUG_WARN,
 925         "Failed to update capability: [%lu] %016lx - %016lx (%016lx -> %016lx)\r\n",
 926         (UINT64)Index, MemorySpaceMap[Index].BaseAddress,
 927         MemorySpaceMap[Index].BaseAddress + MemorySpaceMap[Index].Length - 1,
 928         MemorySpaceMap[Index].Capabilities,
 929         MemorySpaceMap[Index].Capabilities | Capabilities
 930         ));
 931       continue;
 932     }
 933
 934     if (MemorySpaceMap[Index].BaseAddress >= (BaseAddress + PageLength)) {
 935       //
 936       // Current memory space starts at a new page. Resetting PageLength will
 937       // trigger a retrieval of page attributes at new address.
 938       //
 939       PageLength = 0;
 940     } else {
 941       //
 942       // In case current memory space is not adjacent to last one
 943       //
 944       PageLength -= (MemorySpaceMap[Index].BaseAddress - BaseAddress);
 945     }
 946
 947     //
 948     // Sync actual page attributes to GCD
 949     //
 950     BaseAddress       = MemorySpaceMap[Index].BaseAddress;
 951     MemorySpaceLength = MemorySpaceMap[Index].Length;
 952     while (MemorySpaceLength > 0) {
 953       if (PageLength == 0) {
 954         PageEntry = GetPageTableEntry (&PagingContext, BaseAddress, &PageAttribute);
 955         if (PageEntry == NULL) {
 956           break;
 957         }
 958
 959         //
 960         // Note current memory space might start in the middle of a page
 961         //
 962         PageStartAddress  = (*PageEntry) & (UINT64)PageAttributeToMask(PageAttribute);
 963         PageLength        = PageAttributeToLength (PageAttribute) - (BaseAddress - PageStartAddress);
 964         Attributes        = GetAttributesFromPageEntry (PageEntry);
 965       }
 966
 967       Length = MIN (PageLength, MemorySpaceLength);
 968       if (Attributes != (MemorySpaceMap[Index].Attributes &
 969                          EFI_MEMORY_PAGETYPE_MASK)) {
 970         NewAttributes = (MemorySpaceMap[Index].Attributes &
 971                          ~EFI_MEMORY_PAGETYPE_MASK) | Attributes;
 972         Status = gDS->SetMemorySpaceAttributes (
 973                         BaseAddress,
 974                         Length,
 975                         NewAttributes
 976                         );
 977         ASSERT_EFI_ERROR (Status);
 978         DEBUG ((
 979           DEBUG_VERBOSE,
 980           "Updated memory space attribute: [%lu] %016lx - %016lx (%016lx -> %016lx)\r\n",
 981           (UINT64)Index, BaseAddress, BaseAddress + Length - 1,
 982           MemorySpaceMap[Index].Attributes,
 983           NewAttributes
 984           ));
 985       }
 986
 987       PageLength        -= Length;
 988       MemorySpaceLength -= Length;
 989       BaseAddress       += Length;
 990     }
 991   }
 992
 993   FreePool (MemorySpaceMap);
 994 }
 995
 996 /**
 997   Initialize a buffer pool for page table use only.
 998
 999   To reduce the potential split operation on page table, the pages reserved for
1000   page table should be allocated in the times of PAGE_TABLE_POOL_UNIT_PAGES and
1001   at the boundary of PAGE_TABLE_POOL_ALIGNMENT. So the page pool is always
1002   initialized with number of pages greater than or equal to the given PoolPages.
1003
1004   Once the pages in the pool are used up, this method should be called again to
1005   reserve at least another PAGE_TABLE_POOL_UNIT_PAGES. Usually this won't happen
1006   often in practice.
1007
1008   @param[in] PoolPages      The least page number of the pool to be created.
1009
1010   @retval TRUE    The pool is initialized successfully.
1011   @retval FALSE   The memory is out of resource.
1012 **/
1013 BOOLEAN
1014 InitializePageTablePool (
1015   IN  UINTN                           PoolPages
1016   )
1017 {
1018   VOID                      *Buffer;
1019   BOOLEAN                   IsModified;
1020
1021   //
1022   // Always reserve at least PAGE_TABLE_POOL_UNIT_PAGES, including one page for
1023   // header.
1024   //
1025   PoolPages += 1;   // Add one page for header.
1026   PoolPages = ((PoolPages - 1) / PAGE_TABLE_POOL_UNIT_PAGES + 1) *
1027               PAGE_TABLE_POOL_UNIT_PAGES;
1028   Buffer = AllocateAlignedPages (PoolPages, PAGE_TABLE_POOL_ALIGNMENT);
1029   if (Buffer == NULL) {
1030     DEBUG ((DEBUG_ERROR, "ERROR: Out of aligned pages\r\n"));
1031     return FALSE;
1032   }
1033
1034   //
1035   // Link all pools into a list for easier track later.
1036   //
1037   if (mPageTablePool == NULL) {
1038     mPageTablePool = Buffer;
1039     mPageTablePool->NextPool = mPageTablePool;
1040   } else {
1041     ((PAGE_TABLE_POOL *)Buffer)->NextPool = mPageTablePool->NextPool;
1042     mPageTablePool->NextPool = Buffer;
1043     mPageTablePool = Buffer;
1044   }
1045
1046   //
1047   // Reserve one page for pool header.
1048   //
1049   mPageTablePool->FreePages  = PoolPages - 1;
1050   mPageTablePool->Offset = EFI_PAGES_TO_SIZE (1);
1051
1052   //
1053   // Mark the whole pool pages as read-only.
1054   //
1055   ConvertMemoryPageAttributes (
1056     NULL,
1057     (PHYSICAL_ADDRESS)(UINTN)Buffer,
1058     EFI_PAGES_TO_SIZE (PoolPages),
1059     EFI_MEMORY_RO,
1060     PageActionSet,
1061     AllocatePageTableMemory,
1062     NULL,
1063     &IsModified
1064     );
1065   ASSERT (IsModified == TRUE);
1066
1067   return TRUE;
1068 }
1069
1070 /**
1071   This API provides a way to allocate memory for page table.
1072
1073   This API can be called more than once to allocate memory for page tables.
1074
1075   Allocates the number of 4KB pages and returns a pointer to the allocated
1076   buffer. The buffer returned is aligned on a 4KB boundary.
1077
1078   If Pages is 0, then NULL is returned.
1079   If there is not enough memory remaining to satisfy the request, then NULL is
1080   returned.
1081
1082   @param  Pages                 The number of 4 KB pages to allocate.
1083
1084   @return A pointer to the allocated buffer or NULL if allocation fails.
1085
1086 **/
1087 VOID *
1088 EFIAPI
1089 AllocatePageTableMemory (
1090   IN UINTN           Pages
1091   )
1092 {
1093   VOID                            *Buffer;
1094
1095   if (Pages == 0) {
1096     return NULL;
1097   }
1098
1099   //
1100   // Renew the pool if necessary.
1101   //
1102   if (mPageTablePool == NULL ||
1103       Pages > mPageTablePool->FreePages) {
1104     if (!InitializePageTablePool (Pages)) {
1105       return NULL;
1106     }
1107   }
1108
1109   Buffer = (UINT8 *)mPageTablePool + mPageTablePool->Offset;
1110
1111   mPageTablePool->Offset     += EFI_PAGES_TO_SIZE (Pages);
1112   mPageTablePool->FreePages  -= Pages;
1113
1114   return Buffer;
1115 }
1116
1117 /**
1118   Initialize the Page Table lib.
1119 **/
1120 VOID
1121 InitializePageTableLib (
1122   VOID
1123   )
1124 {
1125   PAGE_TABLE_LIB_PAGING_CONTEXT     CurrentPagingContext;
1126
1127   GetCurrentPagingContext (&CurrentPagingContext);
1128
1129   //
1130   // Reserve memory of page tables for future uses, if paging is enabled.
1131   //
1132   if (CurrentPagingContext.ContextData.X64.PageTableBase != 0 &&
1133       (CurrentPagingContext.ContextData.Ia32.Attributes &
1134        PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE) != 0) {
1135     DisableReadOnlyPageWriteProtect ();
1136     InitializePageTablePool (1);
1137     EnableReadOnlyPageWriteProtect ();
1138   }
1139
1140   DEBUG ((DEBUG_INFO, "CurrentPagingContext:\n", CurrentPagingContext.MachineType));
1141   DEBUG ((DEBUG_INFO, "  MachineType   - 0x%x\n", CurrentPagingContext.MachineType));
1142   DEBUG ((DEBUG_INFO, "  PageTableBase - 0x%x\n", CurrentPagingContext.ContextData.X64.PageTableBase));
1143   DEBUG ((DEBUG_INFO, "  Attributes    - 0x%x\n", CurrentPagingContext.ContextData.X64.Attributes));
1144
1145   return ;
1146 }
1147