UefiCpuPkg/CpuDxe/CpuPageTable.c

   1 /** @file
   2   Page table management support.
   3
   4   Copyright (c) 2017 - 2019, Intel Corporation. All rights reserved.<BR>
   5   Copyright (c) 2017, AMD Incorporated. All rights reserved.<BR>
   6
   7   SPDX-License-Identifier: BSD-2-Clause-Patent
   8
   9 **/
  10
  11 #include <Base.h>
  12 #include <Uefi.h>
  13 #include <Library/PeCoffGetEntryPointLib.h>
  14 #include <Library/SerialPortLib.h>
  15 #include <Library/SynchronizationLib.h>
  16 #include <Library/PrintLib.h>
  17 #include <Protocol/SmmBase2.h>
  18 #include <Register/Cpuid.h>
  19 #include <Register/Msr.h>
  20
  21 #include "CpuDxe.h"
  22 #include "CpuPageTable.h"
  23
  24 ///
  25 /// Page Table Entry
  26 ///
  27 #define IA32_PG_P                   BIT0
  28 #define IA32_PG_RW                  BIT1
  29 #define IA32_PG_U                   BIT2
  30 #define IA32_PG_WT                  BIT3
  31 #define IA32_PG_CD                  BIT4
  32 #define IA32_PG_A                   BIT5
  33 #define IA32_PG_D                   BIT6
  34 #define IA32_PG_PS                  BIT7
  35 #define IA32_PG_PAT_2M              BIT12
  36 #define IA32_PG_PAT_4K              IA32_PG_PS
  37 #define IA32_PG_PMNT                BIT62
  38 #define IA32_PG_NX                  BIT63
  39
  40 #define PAGE_ATTRIBUTE_BITS         (IA32_PG_D | IA32_PG_A | IA32_PG_U | IA32_PG_RW | IA32_PG_P)
  41 //
  42 // Bits 1, 2, 5, 6 are reserved in the IA32 PAE PDPTE
  43 // X64 PAE PDPTE does not have such restriction
  44 //
  45 #define IA32_PAE_PDPTE_ATTRIBUTE_BITS    (IA32_PG_P)
  46
  47 #define PAGE_PROGATE_BITS           (IA32_PG_NX | PAGE_ATTRIBUTE_BITS)
  48
  49 #define PAGING_4K_MASK  0xFFF
  50 #define PAGING_2M_MASK  0x1FFFFF
  51 #define PAGING_1G_MASK  0x3FFFFFFF
  52
  53 #define PAGING_PAE_INDEX_MASK  0x1FF
  54
  55 #define PAGING_4K_ADDRESS_MASK_64 0x000FFFFFFFFFF000ull
  56 #define PAGING_2M_ADDRESS_MASK_64 0x000FFFFFFFE00000ull
  57 #define PAGING_1G_ADDRESS_MASK_64 0x000FFFFFC0000000ull
  58
  59 #define MAX_PF_ENTRY_COUNT        10
  60 #define MAX_DEBUG_MESSAGE_LENGTH  0x100
  61 #define IA32_PF_EC_ID             BIT4
  62
  63 typedef enum {
  64   PageNone,
  65   Page4K,
  66   Page2M,
  67   Page1G,
  68 } PAGE_ATTRIBUTE;
  69
  70 typedef struct {
  71   PAGE_ATTRIBUTE   Attribute;
  72   UINT64           Length;
  73   UINT64           AddressMask;
  74 } PAGE_ATTRIBUTE_TABLE;
  75
  76 typedef enum {
  77   PageActionAssign,
  78   PageActionSet,
  79   PageActionClear,
  80 } PAGE_ACTION;
  81
  82 PAGE_ATTRIBUTE_TABLE mPageAttributeTable[] = {
  83   {Page4K,  SIZE_4KB, PAGING_4K_ADDRESS_MASK_64},
  84   {Page2M,  SIZE_2MB, PAGING_2M_ADDRESS_MASK_64},
  85   {Page1G,  SIZE_1GB, PAGING_1G_ADDRESS_MASK_64},
  86 };
  87
  88 PAGE_TABLE_POOL                   *mPageTablePool = NULL;
  89 BOOLEAN                           mPageTablePoolLock = FALSE;
  90 PAGE_TABLE_LIB_PAGING_CONTEXT     mPagingContext;
  91 EFI_SMM_BASE2_PROTOCOL            *mSmmBase2 = NULL;
  92
  93 //
  94 // Record the page fault exception count for one instruction execution.
  95 //
  96 UINTN                     *mPFEntryCount;
  97 UINT64                    *(*mLastPFEntryPointer)[MAX_PF_ENTRY_COUNT];
  98
  99 /**
 100  Check if current execution environment is in SMM mode or not, via
 101  EFI_SMM_BASE2_PROTOCOL.
 102
 103  This is necessary because of the fact that MdePkg\Library\SmmMemoryAllocationLib
 104  supports to free memory outside SMRAM. The library will call gBS->FreePool() or
 105  gBS->FreePages() and then SetMemorySpaceAttributes interface in turn to change
 106  memory paging attributes during free operation, if some memory related features
 107  are enabled (like Heap Guard).
 108
 109  This means that SetMemorySpaceAttributes() has chance to run in SMM mode. This
 110  will cause incorrect result because SMM mode always loads its own page tables,
 111  which are usually different from DXE. This function can be used to detect such
 112  situation and help to avoid further misoperations.
 113
 114   @retval TRUE    In SMM mode.
 115   @retval FALSE   Not in SMM mode.
 116 **/
 117 BOOLEAN
 118 IsInSmm (
 119   VOID
 120   )
 121 {
 122   BOOLEAN                 InSmm;
 123
 124   InSmm = FALSE;
 125   if (mSmmBase2 == NULL) {
 126     gBS->LocateProtocol (&gEfiSmmBase2ProtocolGuid, NULL, (VOID **)&mSmmBase2);
 127   }
 128
 129   if (mSmmBase2 != NULL) {
 130     mSmmBase2->InSmm (mSmmBase2, &InSmm);
 131   }
 132
 133   //
 134   // mSmmBase2->InSmm() can only detect if the caller is running in SMRAM
 135   // or from SMM driver. It cannot tell if the caller is running in SMM mode.
 136   // Check page table base address to guarantee that because SMM mode willl
 137   // load its own page table.
 138   //
 139   return (InSmm &&
 140           mPagingContext.ContextData.X64.PageTableBase != (UINT64)AsmReadCr3());
 141 }
 142
 143 /**
 144   Return current paging context.
 145
 146   @param[in,out]  PagingContext     The paging context.
 147 **/
 148 VOID
 149 GetCurrentPagingContext (
 150   IN OUT PAGE_TABLE_LIB_PAGING_CONTEXT     *PagingContext
 151   )
 152 {
 153   UINT32                          RegEax;
 154   CPUID_EXTENDED_CPU_SIG_EDX      RegEdx;
 155   MSR_IA32_EFER_REGISTER          MsrEfer;
 156   IA32_CR4                        Cr4;
 157   IA32_CR0                        Cr0;
 158
 159   //
 160   // Don't retrieve current paging context from processor if in SMM mode.
 161   //
 162   if (!IsInSmm ()) {
 163     ZeroMem (&mPagingContext, sizeof(mPagingContext));
 164     if (sizeof(UINTN) == sizeof(UINT64)) {
 165       mPagingContext.MachineType = IMAGE_FILE_MACHINE_X64;
 166     } else {
 167       mPagingContext.MachineType = IMAGE_FILE_MACHINE_I386;
 168     }
 169
 170     Cr0.UintN = AsmReadCr0 ();
 171     Cr4.UintN = AsmReadCr4 ();
 172
 173     if (Cr0.Bits.PG != 0) {
 174       mPagingContext.ContextData.X64.PageTableBase = (AsmReadCr3 () & PAGING_4K_ADDRESS_MASK_64);
 175     } else {
 176       mPagingContext.ContextData.X64.PageTableBase = 0;
 177     }
 178     if (Cr0.Bits.WP  != 0) {
 179       mPagingContext.ContextData.Ia32.Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_WP_ENABLE;
 180     }
 181     if (Cr4.Bits.PSE != 0) {
 182       mPagingContext.ContextData.Ia32.Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PSE;
 183     }
 184     if (Cr4.Bits.PAE != 0) {
 185       mPagingContext.ContextData.Ia32.Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE;
 186     }
 187
 188     AsmCpuid (CPUID_EXTENDED_FUNCTION, &RegEax, NULL, NULL, NULL);
 189     if (RegEax >= CPUID_EXTENDED_CPU_SIG) {
 190       AsmCpuid (CPUID_EXTENDED_CPU_SIG, NULL, NULL, NULL, &RegEdx.Uint32);
 191
 192       if (RegEdx.Bits.NX != 0) {
 193         // XD supported
 194         MsrEfer.Uint64 = AsmReadMsr64(MSR_CORE_IA32_EFER);
 195         if (MsrEfer.Bits.NXE != 0) {
 196           // XD activated
 197           mPagingContext.ContextData.Ia32.Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_XD_ACTIVATED;
 198         }
 199       }
 200
 201       if (RegEdx.Bits.Page1GB != 0) {
 202         mPagingContext.ContextData.Ia32.Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAGE_1G_SUPPORT;
 203       }
 204     }
 205   }
 206
 207   //
 208   // This can avoid getting SMM paging context if in SMM mode. We cannot assume
 209   // SMM mode shares the same paging context as DXE.
 210   //
 211   CopyMem (PagingContext, &mPagingContext, sizeof (mPagingContext));
 212 }
 213
 214 /**
 215   Return length according to page attributes.
 216
 217   @param[in]  PageAttributes   The page attribute of the page entry.
 218
 219   @return The length of page entry.
 220 **/
 221 UINTN
 222 PageAttributeToLength (
 223   IN PAGE_ATTRIBUTE  PageAttribute
 224   )
 225 {
 226   UINTN  Index;
 227   for (Index = 0; Index < sizeof(mPageAttributeTable)/sizeof(mPageAttributeTable[0]); Index++) {
 228     if (PageAttribute == mPageAttributeTable[Index].Attribute) {
 229       return (UINTN)mPageAttributeTable[Index].Length;
 230     }
 231   }
 232   return 0;
 233 }
 234
 235 /**
 236   Return address mask according to page attributes.
 237
 238   @param[in]  PageAttributes   The page attribute of the page entry.
 239
 240   @return The address mask of page entry.
 241 **/
 242 UINTN
 243 PageAttributeToMask (
 244   IN PAGE_ATTRIBUTE  PageAttribute
 245   )
 246 {
 247   UINTN  Index;
 248   for (Index = 0; Index < sizeof(mPageAttributeTable)/sizeof(mPageAttributeTable[0]); Index++) {
 249     if (PageAttribute == mPageAttributeTable[Index].Attribute) {
 250       return (UINTN)mPageAttributeTable[Index].AddressMask;
 251     }
 252   }
 253   return 0;
 254 }
 255
 256 /**
 257   Return page table entry to match the address.
 258
 259   @param[in]  PagingContext     The paging context.
 260   @param[in]  Address           The address to be checked.
 261   @param[out] PageAttributes    The page attribute of the page entry.
 262
 263   @return The page entry.
 264 **/
 265 VOID *
 266 GetPageTableEntry (
 267   IN  PAGE_TABLE_LIB_PAGING_CONTEXT     *PagingContext,
 268   IN  PHYSICAL_ADDRESS                  Address,
 269   OUT PAGE_ATTRIBUTE                    *PageAttribute
 270   )
 271 {
 272   UINTN                 Index1;
 273   UINTN                 Index2;
 274   UINTN                 Index3;
 275   UINTN                 Index4;
 276   UINT64                *L1PageTable;
 277   UINT64                *L2PageTable;
 278   UINT64                *L3PageTable;
 279   UINT64                *L4PageTable;
 280   UINT64                AddressEncMask;
 281
 282   ASSERT (PagingContext != NULL);
 283
 284   Index4 = ((UINTN)RShiftU64 (Address, 39)) & PAGING_PAE_INDEX_MASK;
 285   Index3 = ((UINTN)Address >> 30) & PAGING_PAE_INDEX_MASK;
 286   Index2 = ((UINTN)Address >> 21) & PAGING_PAE_INDEX_MASK;
 287   Index1 = ((UINTN)Address >> 12) & PAGING_PAE_INDEX_MASK;
 288
 289   // Make sure AddressEncMask is contained to smallest supported address field.
 290   //
 291   AddressEncMask = PcdGet64 (PcdPteMemoryEncryptionAddressOrMask) & PAGING_1G_ADDRESS_MASK_64;
 292
 293   if (PagingContext->MachineType == IMAGE_FILE_MACHINE_X64) {
 294     L4PageTable = (UINT64 *)(UINTN)PagingContext->ContextData.X64.PageTableBase;
 295     if (L4PageTable[Index4] == 0) {
 296       *PageAttribute = PageNone;
 297       return NULL;
 298     }
 299
 300     L3PageTable = (UINT64 *)(UINTN)(L4PageTable[Index4] & ~AddressEncMask & PAGING_4K_ADDRESS_MASK_64);
 301   } else {
 302     ASSERT((PagingContext->ContextData.Ia32.Attributes & PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE) != 0);
 303     L3PageTable = (UINT64 *)(UINTN)PagingContext->ContextData.Ia32.PageTableBase;
 304   }
 305   if (L3PageTable[Index3] == 0) {
 306     *PageAttribute = PageNone;
 307     return NULL;
 308   }
 309   if ((L3PageTable[Index3] & IA32_PG_PS) != 0) {
 310     // 1G
 311     *PageAttribute = Page1G;
 312     return &L3PageTable[Index3];
 313   }
 314
 315   L2PageTable = (UINT64 *)(UINTN)(L3PageTable[Index3] & ~AddressEncMask & PAGING_4K_ADDRESS_MASK_64);
 316   if (L2PageTable[Index2] == 0) {
 317     *PageAttribute = PageNone;
 318     return NULL;
 319   }
 320   if ((L2PageTable[Index2] & IA32_PG_PS) != 0) {
 321     // 2M
 322     *PageAttribute = Page2M;
 323     return &L2PageTable[Index2];
 324   }
 325
 326   // 4k
 327   L1PageTable = (UINT64 *)(UINTN)(L2PageTable[Index2] & ~AddressEncMask & PAGING_4K_ADDRESS_MASK_64);
 328   if ((L1PageTable[Index1] == 0) && (Address != 0)) {
 329     *PageAttribute = PageNone;
 330     return NULL;
 331   }
 332   *PageAttribute = Page4K;
 333   return &L1PageTable[Index1];
 334 }
 335
 336 /**
 337   Return memory attributes of page entry.
 338
 339   @param[in]  PageEntry        The page entry.
 340
 341   @return Memory attributes of page entry.
 342 **/
 343 UINT64
 344 GetAttributesFromPageEntry (
 345   IN  UINT64                            *PageEntry
 346   )
 347 {
 348   UINT64  Attributes;
 349   Attributes = 0;
 350   if ((*PageEntry & IA32_PG_P) == 0) {
 351     Attributes |= EFI_MEMORY_RP;
 352   }
 353   if ((*PageEntry & IA32_PG_RW) == 0) {
 354     Attributes |= EFI_MEMORY_RO;
 355   }
 356   if ((*PageEntry & IA32_PG_NX) != 0) {
 357     Attributes |= EFI_MEMORY_XP;
 358   }
 359   return Attributes;
 360 }
 361
 362 /**
 363   Modify memory attributes of page entry.
 364
 365   @param[in]  PagingContext    The paging context.
 366   @param[in]  PageEntry        The page entry.
 367   @param[in]  Attributes       The bit mask of attributes to modify for the memory region.
 368   @param[in]  PageAction       The page action.
 369   @param[out] IsModified       TRUE means page table modified. FALSE means page table not modified.
 370 **/
 371 VOID
 372 ConvertPageEntryAttribute (
 373   IN  PAGE_TABLE_LIB_PAGING_CONTEXT     *PagingContext,
 374   IN  UINT64                            *PageEntry,
 375   IN  UINT64                            Attributes,
 376   IN  PAGE_ACTION                       PageAction,
 377   OUT BOOLEAN                           *IsModified
 378   )
 379 {
 380   UINT64  CurrentPageEntry;
 381   UINT64  NewPageEntry;
 382
 383   CurrentPageEntry = *PageEntry;
 384   NewPageEntry = CurrentPageEntry;
 385   if ((Attributes & EFI_MEMORY_RP) != 0) {
 386     switch (PageAction) {
 387     case PageActionAssign:
 388     case PageActionSet:
 389       NewPageEntry &= ~(UINT64)IA32_PG_P;
 390       break;
 391     case PageActionClear:
 392       NewPageEntry |= IA32_PG_P;
 393       break;
 394     }
 395   } else {
 396     switch (PageAction) {
 397     case PageActionAssign:
 398       NewPageEntry |= IA32_PG_P;
 399       break;
 400     case PageActionSet:
 401     case PageActionClear:
 402       break;
 403     }
 404   }
 405   if ((Attributes & EFI_MEMORY_RO) != 0) {
 406     switch (PageAction) {
 407     case PageActionAssign:
 408     case PageActionSet:
 409       NewPageEntry &= ~(UINT64)IA32_PG_RW;
 410       break;
 411     case PageActionClear:
 412       NewPageEntry |= IA32_PG_RW;
 413       break;
 414     }
 415   } else {
 416     switch (PageAction) {
 417     case PageActionAssign:
 418       NewPageEntry |= IA32_PG_RW;
 419       break;
 420     case PageActionSet:
 421     case PageActionClear:
 422       break;
 423     }
 424   }
 425   if ((PagingContext->ContextData.Ia32.Attributes & PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_XD_ACTIVATED) != 0) {
 426     if ((Attributes & EFI_MEMORY_XP) != 0) {
 427       switch (PageAction) {
 428       case PageActionAssign:
 429       case PageActionSet:
 430         NewPageEntry |= IA32_PG_NX;
 431         break;
 432       case PageActionClear:
 433         NewPageEntry &= ~IA32_PG_NX;
 434         break;
 435       }
 436     } else {
 437       switch (PageAction) {
 438       case PageActionAssign:
 439         NewPageEntry &= ~IA32_PG_NX;
 440         break;
 441       case PageActionSet:
 442       case PageActionClear:
 443         break;
 444       }
 445     }
 446   }
 447   *PageEntry = NewPageEntry;
 448   if (CurrentPageEntry != NewPageEntry) {
 449     *IsModified = TRUE;
 450     DEBUG ((DEBUG_VERBOSE, "ConvertPageEntryAttribute 0x%lx", CurrentPageEntry));
 451     DEBUG ((DEBUG_VERBOSE, "->0x%lx\n", NewPageEntry));
 452   } else {
 453     *IsModified = FALSE;
 454   }
 455 }
 456
 457 /**
 458   This function returns if there is need to split page entry.
 459
 460   @param[in]  BaseAddress      The base address to be checked.
 461   @param[in]  Length           The length to be checked.
 462   @param[in]  PageEntry        The page entry to be checked.
 463   @param[in]  PageAttribute    The page attribute of the page entry.
 464
 465   @retval SplitAttributes on if there is need to split page entry.
 466 **/
 467 PAGE_ATTRIBUTE
 468 NeedSplitPage (
 469   IN  PHYSICAL_ADDRESS                  BaseAddress,
 470   IN  UINT64                            Length,
 471   IN  UINT64                            *PageEntry,
 472   IN  PAGE_ATTRIBUTE                    PageAttribute
 473   )
 474 {
 475   UINT64                PageEntryLength;
 476
 477   PageEntryLength = PageAttributeToLength (PageAttribute);
 478
 479   if (((BaseAddress & (PageEntryLength - 1)) == 0) && (Length >= PageEntryLength)) {
 480     return PageNone;
 481   }
 482
 483   if (((BaseAddress & PAGING_2M_MASK) != 0) || (Length < SIZE_2MB)) {
 484     return Page4K;
 485   }
 486
 487   return Page2M;
 488 }
 489
 490 /**
 491   This function splits one page entry to small page entries.
 492
 493   @param[in]  PageEntry         The page entry to be splitted.
 494   @param[in]  PageAttribute     The page attribute of the page entry.
 495   @param[in]  SplitAttribute    How to split the page entry.
 496   @param[in]  AllocatePagesFunc If page split is needed, this function is used to allocate more pages.
 497
 498   @retval RETURN_SUCCESS            The page entry is splitted.
 499   @retval RETURN_UNSUPPORTED        The page entry does not support to be splitted.
 500   @retval RETURN_OUT_OF_RESOURCES   No resource to split page entry.
 501 **/
 502 RETURN_STATUS
 503 SplitPage (
 504   IN  UINT64                            *PageEntry,
 505   IN  PAGE_ATTRIBUTE                    PageAttribute,
 506   IN  PAGE_ATTRIBUTE                    SplitAttribute,
 507   IN  PAGE_TABLE_LIB_ALLOCATE_PAGES     AllocatePagesFunc
 508   )
 509 {
 510   UINT64   BaseAddress;
 511   UINT64   *NewPageEntry;
 512   UINTN    Index;
 513   UINT64   AddressEncMask;
 514
 515   ASSERT (PageAttribute == Page2M || PageAttribute == Page1G);
 516
 517   ASSERT (AllocatePagesFunc != NULL);
 518
 519   // Make sure AddressEncMask is contained to smallest supported address field.
 520   //
 521   AddressEncMask = PcdGet64 (PcdPteMemoryEncryptionAddressOrMask) & PAGING_1G_ADDRESS_MASK_64;
 522
 523   if (PageAttribute == Page2M) {
 524     //
 525     // Split 2M to 4K
 526     //
 527     ASSERT (SplitAttribute == Page4K);
 528     if (SplitAttribute == Page4K) {
 529       NewPageEntry = AllocatePagesFunc (1);
 530       DEBUG ((DEBUG_VERBOSE, "Split - 0x%x\n", NewPageEntry));
 531       if (NewPageEntry == NULL) {
 532         return RETURN_OUT_OF_RESOURCES;
 533       }
 534       BaseAddress = *PageEntry & ~AddressEncMask & PAGING_2M_ADDRESS_MASK_64;
 535       for (Index = 0; Index < SIZE_4KB / sizeof(UINT64); Index++) {
 536         NewPageEntry[Index] = (BaseAddress + SIZE_4KB * Index) | AddressEncMask | ((*PageEntry) & PAGE_PROGATE_BITS);
 537       }
 538       (*PageEntry) = (UINT64)(UINTN)NewPageEntry | AddressEncMask | ((*PageEntry) & PAGE_ATTRIBUTE_BITS);
 539       return RETURN_SUCCESS;
 540     } else {
 541       return RETURN_UNSUPPORTED;
 542     }
 543   } else if (PageAttribute == Page1G) {
 544     //
 545     // Split 1G to 2M
 546     // No need support 1G->4K directly, we should use 1G->2M, then 2M->4K to get more compact page table.
 547     //
 548     ASSERT (SplitAttribute == Page2M || SplitAttribute == Page4K);
 549     if ((SplitAttribute == Page2M || SplitAttribute == Page4K)) {
 550       NewPageEntry = AllocatePagesFunc (1);
 551       DEBUG ((DEBUG_VERBOSE, "Split - 0x%x\n", NewPageEntry));
 552       if (NewPageEntry == NULL) {
 553         return RETURN_OUT_OF_RESOURCES;
 554       }
 555       BaseAddress = *PageEntry & ~AddressEncMask  & PAGING_1G_ADDRESS_MASK_64;
 556       for (Index = 0; Index < SIZE_4KB / sizeof(UINT64); Index++) {
 557         NewPageEntry[Index] = (BaseAddress + SIZE_2MB * Index) | AddressEncMask | IA32_PG_PS | ((*PageEntry) & PAGE_PROGATE_BITS);
 558       }
 559       (*PageEntry) = (UINT64)(UINTN)NewPageEntry | AddressEncMask | ((*PageEntry) & PAGE_ATTRIBUTE_BITS);
 560       return RETURN_SUCCESS;
 561     } else {
 562       return RETURN_UNSUPPORTED;
 563     }
 564   } else {
 565     return RETURN_UNSUPPORTED;
 566   }
 567 }
 568
 569 /**
 570  Check the WP status in CR0 register. This bit is used to lock or unlock write
 571  access to pages marked as read-only.
 572
 573   @retval TRUE    Write protection is enabled.
 574   @retval FALSE   Write protection is disabled.
 575 **/
 576 BOOLEAN
 577 IsReadOnlyPageWriteProtected (
 578   VOID
 579   )
 580 {
 581   IA32_CR0  Cr0;
 582   //
 583   // To avoid unforseen consequences, don't touch paging settings in SMM mode
 584   // in this driver.
 585   //
 586   if (!IsInSmm ()) {
 587     Cr0.UintN = AsmReadCr0 ();
 588     return (BOOLEAN) (Cr0.Bits.WP != 0);
 589   }
 590   return FALSE;
 591 }
 592
 593 /**
 594  Disable Write Protect on pages marked as read-only.
 595 **/
 596 VOID
 597 DisableReadOnlyPageWriteProtect (
 598   VOID
 599   )
 600 {
 601   IA32_CR0  Cr0;
 602   //
 603   // To avoid unforseen consequences, don't touch paging settings in SMM mode
 604   // in this driver.
 605   //
 606   if (!IsInSmm ()) {
 607     Cr0.UintN = AsmReadCr0 ();
 608     Cr0.Bits.WP = 0;
 609     AsmWriteCr0 (Cr0.UintN);
 610   }
 611 }
 612
 613 /**
 614  Enable Write Protect on pages marked as read-only.
 615 **/
 616 VOID
 617 EnableReadOnlyPageWriteProtect (
 618   VOID
 619   )
 620 {
 621   IA32_CR0  Cr0;
 622   //
 623   // To avoid unforseen consequences, don't touch paging settings in SMM mode
 624   // in this driver.
 625   //
 626   if (!IsInSmm ()) {
 627     Cr0.UintN = AsmReadCr0 ();
 628     Cr0.Bits.WP = 1;
 629     AsmWriteCr0 (Cr0.UintN);
 630   }
 631 }
 632
 633 /**
 634   This function modifies the page attributes for the memory region specified by BaseAddress and
 635   Length from their current attributes to the attributes specified by Attributes.
 636
 637   Caller should make sure BaseAddress and Length is at page boundary.
 638
 639   @param[in]  PagingContext     The paging context. NULL means get page table from current CPU context.
 640   @param[in]  BaseAddress       The physical address that is the start address of a memory region.
 641   @param[in]  Length            The size in bytes of the memory region.
 642   @param[in]  Attributes        The bit mask of attributes to modify for the memory region.
 643   @param[in]  PageAction        The page action.
 644   @param[in]  AllocatePagesFunc If page split is needed, this function is used to allocate more pages.
 645                                 NULL mean page split is unsupported.
 646   @param[out] IsSplitted        TRUE means page table splitted. FALSE means page table not splitted.
 647   @param[out] IsModified        TRUE means page table modified. FALSE means page table not modified.
 648
 649   @retval RETURN_SUCCESS           The attributes were modified for the memory region.
 650   @retval RETURN_ACCESS_DENIED     The attributes for the memory resource range specified by
 651                                    BaseAddress and Length cannot be modified.
 652   @retval RETURN_INVALID_PARAMETER Length is zero.
 653                                    Attributes specified an illegal combination of attributes that
 654                                    cannot be set together.
 655   @retval RETURN_OUT_OF_RESOURCES  There are not enough system resources to modify the attributes of
 656                                    the memory resource range.
 657   @retval RETURN_UNSUPPORTED       The processor does not support one or more bytes of the memory
 658                                    resource range specified by BaseAddress and Length.
 659                                    The bit mask of attributes is not support for the memory resource
 660                                    range specified by BaseAddress and Length.
 661 **/
 662 RETURN_STATUS
 663 ConvertMemoryPageAttributes (
 664   IN  PAGE_TABLE_LIB_PAGING_CONTEXT     *PagingContext OPTIONAL,
 665   IN  PHYSICAL_ADDRESS                  BaseAddress,
 666   IN  UINT64                            Length,
 667   IN  UINT64                            Attributes,
 668   IN  PAGE_ACTION                       PageAction,
 669   IN  PAGE_TABLE_LIB_ALLOCATE_PAGES     AllocatePagesFunc OPTIONAL,
 670   OUT BOOLEAN                           *IsSplitted,  OPTIONAL
 671   OUT BOOLEAN                           *IsModified   OPTIONAL
 672   )
 673 {
 674   PAGE_TABLE_LIB_PAGING_CONTEXT     CurrentPagingContext;
 675   UINT64                            *PageEntry;
 676   PAGE_ATTRIBUTE                    PageAttribute;
 677   UINTN                             PageEntryLength;
 678   PAGE_ATTRIBUTE                    SplitAttribute;
 679   RETURN_STATUS                     Status;
 680   BOOLEAN                           IsEntryModified;
 681   BOOLEAN                           IsWpEnabled;
 682
 683   if ((BaseAddress & (SIZE_4KB - 1)) != 0) {
 684     DEBUG ((DEBUG_ERROR, "BaseAddress(0x%lx) is not aligned!\n", BaseAddress));
 685     return EFI_UNSUPPORTED;
 686   }
 687   if ((Length & (SIZE_4KB - 1)) != 0) {
 688     DEBUG ((DEBUG_ERROR, "Length(0x%lx) is not aligned!\n", Length));
 689     return EFI_UNSUPPORTED;
 690   }
 691   if (Length == 0) {
 692     DEBUG ((DEBUG_ERROR, "Length is 0!\n"));
 693     return RETURN_INVALID_PARAMETER;
 694   }
 695
 696   if ((Attributes & ~(EFI_MEMORY_RP | EFI_MEMORY_RO | EFI_MEMORY_XP)) != 0) {
 697     DEBUG ((DEBUG_ERROR, "Attributes(0x%lx) has unsupported bit\n", Attributes));
 698     return EFI_UNSUPPORTED;
 699   }
 700
 701   if (PagingContext == NULL) {
 702     GetCurrentPagingContext (&CurrentPagingContext);
 703   } else {
 704     CopyMem (&CurrentPagingContext, PagingContext, sizeof(CurrentPagingContext));
 705   }
 706   switch(CurrentPagingContext.MachineType) {
 707   case IMAGE_FILE_MACHINE_I386:
 708     if (CurrentPagingContext.ContextData.Ia32.PageTableBase == 0) {
 709       if (Attributes == 0) {
 710         return EFI_SUCCESS;
 711       } else {
 712         DEBUG ((DEBUG_ERROR, "PageTable is 0!\n"));
 713         return EFI_UNSUPPORTED;
 714       }
 715     }
 716     if ((CurrentPagingContext.ContextData.Ia32.Attributes & PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE) == 0) {
 717       DEBUG ((DEBUG_ERROR, "Non-PAE Paging!\n"));
 718       return EFI_UNSUPPORTED;
 719     }
 720     if ((BaseAddress + Length) > BASE_4GB) {
 721       DEBUG ((DEBUG_ERROR, "Beyond 4GB memory in 32-bit mode!\n"));
 722       return EFI_UNSUPPORTED;
 723     }
 724     break;
 725   case IMAGE_FILE_MACHINE_X64:
 726     ASSERT (CurrentPagingContext.ContextData.X64.PageTableBase != 0);
 727     break;
 728   default:
 729     ASSERT(FALSE);
 730     return EFI_UNSUPPORTED;
 731     break;
 732   }
 733
 734 //  DEBUG ((DEBUG_ERROR, "ConvertMemoryPageAttributes(%x) - %016lx, %016lx, %02lx\n", IsSet, BaseAddress, Length, Attributes));
 735
 736   if (IsSplitted != NULL) {
 737     *IsSplitted = FALSE;
 738   }
 739   if (IsModified != NULL) {
 740     *IsModified = FALSE;
 741   }
 742   if (AllocatePagesFunc == NULL) {
 743     AllocatePagesFunc = AllocatePageTableMemory;
 744   }
 745
 746   //
 747   // Make sure that the page table is changeable.
 748   //
 749   IsWpEnabled = IsReadOnlyPageWriteProtected ();
 750   if (IsWpEnabled) {
 751     DisableReadOnlyPageWriteProtect ();
 752   }
 753
 754   //
 755   // Below logic is to check 2M/4K page to make sure we donot waist memory.
 756   //
 757   Status = EFI_SUCCESS;
 758   while (Length != 0) {
 759     PageEntry = GetPageTableEntry (&CurrentPagingContext, BaseAddress, &PageAttribute);
 760     if (PageEntry == NULL) {
 761       Status = RETURN_UNSUPPORTED;
 762       goto Done;
 763     }
 764     PageEntryLength = PageAttributeToLength (PageAttribute);
 765     SplitAttribute = NeedSplitPage (BaseAddress, Length, PageEntry, PageAttribute);
 766     if (SplitAttribute == PageNone) {
 767       ConvertPageEntryAttribute (&CurrentPagingContext, PageEntry, Attributes, PageAction, &IsEntryModified);
 768       if (IsEntryModified) {
 769         if (IsModified != NULL) {
 770           *IsModified = TRUE;
 771         }
 772       }
 773       //
 774       // Convert success, move to next
 775       //
 776       BaseAddress += PageEntryLength;
 777       Length -= PageEntryLength;
 778     } else {
 779       if (AllocatePagesFunc == NULL) {
 780         Status = RETURN_UNSUPPORTED;
 781         goto Done;
 782       }
 783       Status = SplitPage (PageEntry, PageAttribute, SplitAttribute, AllocatePagesFunc);
 784       if (RETURN_ERROR (Status)) {
 785         Status = RETURN_UNSUPPORTED;
 786         goto Done;
 787       }
 788       if (IsSplitted != NULL) {
 789         *IsSplitted = TRUE;
 790       }
 791       if (IsModified != NULL) {
 792         *IsModified = TRUE;
 793       }
 794       //
 795       // Just split current page
 796       // Convert success in next around
 797       //
 798     }
 799   }
 800
 801 Done:
 802   //
 803   // Restore page table write protection, if any.
 804   //
 805   if (IsWpEnabled) {
 806     EnableReadOnlyPageWriteProtect ();
 807   }
 808   return Status;
 809 }
 810
 811 /**
 812   This function assigns the page attributes for the memory region specified by BaseAddress and
 813   Length from their current attributes to the attributes specified by Attributes.
 814
 815   Caller should make sure BaseAddress and Length is at page boundary.
 816
 817   Caller need guarentee the TPL <= TPL_NOTIFY, if there is split page request.
 818
 819   @param[in]  PagingContext     The paging context. NULL means get page table from current CPU context.
 820   @param[in]  BaseAddress       The physical address that is the start address of a memory region.
 821   @param[in]  Length            The size in bytes of the memory region.
 822   @param[in]  Attributes        The bit mask of attributes to set for the memory region.
 823   @param[in]  AllocatePagesFunc If page split is needed, this function is used to allocate more pages.
 824                                 NULL mean page split is unsupported.
 825
 826   @retval RETURN_SUCCESS           The attributes were cleared for the memory region.
 827   @retval RETURN_ACCESS_DENIED     The attributes for the memory resource range specified by
 828                                    BaseAddress and Length cannot be modified.
 829   @retval RETURN_INVALID_PARAMETER Length is zero.
 830                                    Attributes specified an illegal combination of attributes that
 831                                    cannot be set together.
 832   @retval RETURN_OUT_OF_RESOURCES  There are not enough system resources to modify the attributes of
 833                                    the memory resource range.
 834   @retval RETURN_UNSUPPORTED       The processor does not support one or more bytes of the memory
 835                                    resource range specified by BaseAddress and Length.
 836                                    The bit mask of attributes is not support for the memory resource
 837                                    range specified by BaseAddress and Length.
 838 **/
 839 RETURN_STATUS
 840 EFIAPI
 841 AssignMemoryPageAttributes (
 842   IN  PAGE_TABLE_LIB_PAGING_CONTEXT     *PagingContext OPTIONAL,
 843   IN  PHYSICAL_ADDRESS                  BaseAddress,
 844   IN  UINT64                            Length,
 845   IN  UINT64                            Attributes,
 846   IN  PAGE_TABLE_LIB_ALLOCATE_PAGES     AllocatePagesFunc OPTIONAL
 847   )
 848 {
 849   RETURN_STATUS  Status;
 850   BOOLEAN        IsModified;
 851   BOOLEAN        IsSplitted;
 852
 853 //  DEBUG((DEBUG_INFO, "AssignMemoryPageAttributes: 0x%lx - 0x%lx (0x%lx)\n", BaseAddress, Length, Attributes));
 854   Status = ConvertMemoryPageAttributes (PagingContext, BaseAddress, Length, Attributes, PageActionAssign, AllocatePagesFunc, &IsSplitted, &IsModified);
 855   if (!EFI_ERROR(Status)) {
 856     if ((PagingContext == NULL) && IsModified) {
 857       //
 858       // Flush TLB as last step.
 859       //
 860       // Note: Since APs will always init CR3 register in HLT loop mode or do
 861       // TLB flush in MWAIT loop mode, there's no need to flush TLB for them
 862       // here.
 863       //
 864       CpuFlushTlb();
 865     }
 866   }
 867
 868   return Status;
 869 }
 870
 871 /**
 872  Check if Execute Disable feature is enabled or not.
 873 **/
 874 BOOLEAN
 875 IsExecuteDisableEnabled (
 876   VOID
 877   )
 878 {
 879   MSR_CORE_IA32_EFER_REGISTER    MsrEfer;
 880
 881   MsrEfer.Uint64 = AsmReadMsr64 (MSR_IA32_EFER);
 882   return (MsrEfer.Bits.NXE == 1);
 883 }
 884
 885 /**
 886   Update GCD memory space attributes according to current page table setup.
 887 **/
 888 VOID
 889 RefreshGcdMemoryAttributesFromPaging (
 890   VOID
 891   )
 892 {
 893   EFI_STATUS                          Status;
 894   UINTN                               NumberOfDescriptors;
 895   EFI_GCD_MEMORY_SPACE_DESCRIPTOR     *MemorySpaceMap;
 896   PAGE_TABLE_LIB_PAGING_CONTEXT       PagingContext;
 897   PAGE_ATTRIBUTE                      PageAttribute;
 898   UINT64                              *PageEntry;
 899   UINT64                              PageLength;
 900   UINT64                              MemorySpaceLength;
 901   UINT64                              Length;
 902   UINT64                              BaseAddress;
 903   UINT64                              PageStartAddress;
 904   UINT64                              Attributes;
 905   UINT64                              Capabilities;
 906   UINT64                              NewAttributes;
 907   UINTN                               Index;
 908
 909   //
 910   // Assuming that memory space map returned is sorted already; otherwise sort
 911   // them in the order of lowest address to highest address.
 912   //
 913   Status = gDS->GetMemorySpaceMap (&NumberOfDescriptors, &MemorySpaceMap);
 914   ASSERT_EFI_ERROR (Status);
 915
 916   GetCurrentPagingContext (&PagingContext);
 917
 918   Attributes      = 0;
 919   NewAttributes   = 0;
 920   BaseAddress     = 0;
 921   PageLength      = 0;
 922
 923   if (IsExecuteDisableEnabled ()) {
 924     Capabilities = EFI_MEMORY_RO | EFI_MEMORY_RP | EFI_MEMORY_XP;
 925   } else {
 926     Capabilities = EFI_MEMORY_RO | EFI_MEMORY_RP;
 927   }
 928
 929   for (Index = 0; Index < NumberOfDescriptors; Index++) {
 930     if (MemorySpaceMap[Index].GcdMemoryType == EfiGcdMemoryTypeNonExistent) {
 931       continue;
 932     }
 933
 934     //
 935     // Sync the actual paging related capabilities back to GCD service first.
 936     // As a side effect (good one), this can also help to avoid unnecessary
 937     // memory map entries due to the different capabilities of the same type
 938     // memory, such as multiple RT_CODE and RT_DATA entries in memory map,
 939     // which could cause boot failure of some old Linux distro (before v4.3).
 940     //
 941     Status = gDS->SetMemorySpaceCapabilities (
 942                     MemorySpaceMap[Index].BaseAddress,
 943                     MemorySpaceMap[Index].Length,
 944                     MemorySpaceMap[Index].Capabilities | Capabilities
 945                     );
 946     if (EFI_ERROR (Status)) {
 947       //
 948       // If we cannot udpate the capabilities, we cannot update its
 949       // attributes either. So just simply skip current block of memory.
 950       //
 951       DEBUG ((
 952         DEBUG_WARN,
 953         "Failed to update capability: [%lu] %016lx - %016lx (%016lx -> %016lx)\r\n",
 954         (UINT64)Index, MemorySpaceMap[Index].BaseAddress,
 955         MemorySpaceMap[Index].BaseAddress + MemorySpaceMap[Index].Length - 1,
 956         MemorySpaceMap[Index].Capabilities,
 957         MemorySpaceMap[Index].Capabilities | Capabilities
 958         ));
 959       continue;
 960     }
 961
 962     if (MemorySpaceMap[Index].BaseAddress >= (BaseAddress + PageLength)) {
 963       //
 964       // Current memory space starts at a new page. Resetting PageLength will
 965       // trigger a retrieval of page attributes at new address.
 966       //
 967       PageLength = 0;
 968     } else {
 969       //
 970       // In case current memory space is not adjacent to last one
 971       //
 972       PageLength -= (MemorySpaceMap[Index].BaseAddress - BaseAddress);
 973     }
 974
 975     //
 976     // Sync actual page attributes to GCD
 977     //
 978     BaseAddress       = MemorySpaceMap[Index].BaseAddress;
 979     MemorySpaceLength = MemorySpaceMap[Index].Length;
 980     while (MemorySpaceLength > 0) {
 981       if (PageLength == 0) {
 982         PageEntry = GetPageTableEntry (&PagingContext, BaseAddress, &PageAttribute);
 983         if (PageEntry == NULL) {
 984           break;
 985         }
 986
 987         //
 988         // Note current memory space might start in the middle of a page
 989         //
 990         PageStartAddress  = (*PageEntry) & (UINT64)PageAttributeToMask(PageAttribute);
 991         PageLength        = PageAttributeToLength (PageAttribute) - (BaseAddress - PageStartAddress);
 992         Attributes        = GetAttributesFromPageEntry (PageEntry);
 993       }
 994
 995       Length = MIN (PageLength, MemorySpaceLength);
 996       if (Attributes != (MemorySpaceMap[Index].Attributes &
 997                          EFI_MEMORY_PAGETYPE_MASK)) {
 998         NewAttributes = (MemorySpaceMap[Index].Attributes &
 999                          ~EFI_MEMORY_PAGETYPE_MASK) | Attributes;
1000         Status = gDS->SetMemorySpaceAttributes (
1001                         BaseAddress,
1002                         Length,
1003                         NewAttributes
1004                         );
1005         ASSERT_EFI_ERROR (Status);
1006         DEBUG ((
1007           DEBUG_VERBOSE,
1008           "Updated memory space attribute: [%lu] %016lx - %016lx (%016lx -> %016lx)\r\n",
1009           (UINT64)Index, BaseAddress, BaseAddress + Length - 1,
1010           MemorySpaceMap[Index].Attributes,
1011           NewAttributes
1012           ));
1013       }
1014
1015       PageLength        -= Length;
1016       MemorySpaceLength -= Length;
1017       BaseAddress       += Length;
1018     }
1019   }
1020
1021   FreePool (MemorySpaceMap);
1022 }
1023
1024 /**
1025   Initialize a buffer pool for page table use only.
1026
1027   To reduce the potential split operation on page table, the pages reserved for
1028   page table should be allocated in the times of PAGE_TABLE_POOL_UNIT_PAGES and
1029   at the boundary of PAGE_TABLE_POOL_ALIGNMENT. So the page pool is always
1030   initialized with number of pages greater than or equal to the given PoolPages.
1031
1032   Once the pages in the pool are used up, this method should be called again to
1033   reserve at least another PAGE_TABLE_POOL_UNIT_PAGES. Usually this won't happen
1034   often in practice.
1035
1036   @param[in] PoolPages      The least page number of the pool to be created.
1037
1038   @retval TRUE    The pool is initialized successfully.
1039   @retval FALSE   The memory is out of resource.
1040 **/
1041 BOOLEAN
1042 InitializePageTablePool (
1043   IN  UINTN                           PoolPages
1044   )
1045 {
1046   VOID                      *Buffer;
1047   BOOLEAN                   IsModified;
1048
1049   //
1050   // Do not allow re-entrance.
1051   //
1052   if (mPageTablePoolLock) {
1053     return FALSE;
1054   }
1055
1056   mPageTablePoolLock = TRUE;
1057   IsModified = FALSE;
1058
1059   //
1060   // Always reserve at least PAGE_TABLE_POOL_UNIT_PAGES, including one page for
1061   // header.
1062   //
1063   PoolPages += 1;   // Add one page for header.
1064   PoolPages = ((PoolPages - 1) / PAGE_TABLE_POOL_UNIT_PAGES + 1) *
1065               PAGE_TABLE_POOL_UNIT_PAGES;
1066   Buffer = AllocateAlignedPages (PoolPages, PAGE_TABLE_POOL_ALIGNMENT);
1067   if (Buffer == NULL) {
1068     DEBUG ((DEBUG_ERROR, "ERROR: Out of aligned pages\r\n"));
1069     goto Done;
1070   }
1071
1072   DEBUG ((
1073     DEBUG_INFO,
1074     "Paging: added %lu pages to page table pool\r\n",
1075     (UINT64)PoolPages
1076     ));
1077
1078   //
1079   // Link all pools into a list for easier track later.
1080   //
1081   if (mPageTablePool == NULL) {
1082     mPageTablePool = Buffer;
1083     mPageTablePool->NextPool = mPageTablePool;
1084   } else {
1085     ((PAGE_TABLE_POOL *)Buffer)->NextPool = mPageTablePool->NextPool;
1086     mPageTablePool->NextPool = Buffer;
1087     mPageTablePool = Buffer;
1088   }
1089
1090   //
1091   // Reserve one page for pool header.
1092   //
1093   mPageTablePool->FreePages  = PoolPages - 1;
1094   mPageTablePool->Offset = EFI_PAGES_TO_SIZE (1);
1095
1096   //
1097   // Mark the whole pool pages as read-only.
1098   //
1099   ConvertMemoryPageAttributes (
1100     NULL,
1101     (PHYSICAL_ADDRESS)(UINTN)Buffer,
1102     EFI_PAGES_TO_SIZE (PoolPages),
1103     EFI_MEMORY_RO,
1104     PageActionSet,
1105     AllocatePageTableMemory,
1106     NULL,
1107     &IsModified
1108     );
1109   ASSERT (IsModified == TRUE);
1110
1111 Done:
1112   mPageTablePoolLock = FALSE;
1113   return IsModified;
1114 }
1115
1116 /**
1117   This API provides a way to allocate memory for page table.
1118
1119   This API can be called more than once to allocate memory for page tables.
1120
1121   Allocates the number of 4KB pages and returns a pointer to the allocated
1122   buffer. The buffer returned is aligned on a 4KB boundary.
1123
1124   If Pages is 0, then NULL is returned.
1125   If there is not enough memory remaining to satisfy the request, then NULL is
1126   returned.
1127
1128   @param  Pages                 The number of 4 KB pages to allocate.
1129
1130   @return A pointer to the allocated buffer or NULL if allocation fails.
1131
1132 **/
1133 VOID *
1134 EFIAPI
1135 AllocatePageTableMemory (
1136   IN UINTN           Pages
1137   )
1138 {
1139   VOID                            *Buffer;
1140
1141   if (Pages == 0) {
1142     return NULL;
1143   }
1144
1145   //
1146   // Renew the pool if necessary.
1147   //
1148   if (mPageTablePool == NULL ||
1149       Pages > mPageTablePool->FreePages) {
1150     if (!InitializePageTablePool (Pages)) {
1151       return NULL;
1152     }
1153   }
1154
1155   Buffer = (UINT8 *)mPageTablePool + mPageTablePool->Offset;
1156
1157   mPageTablePool->Offset     += EFI_PAGES_TO_SIZE (Pages);
1158   mPageTablePool->FreePages  -= Pages;
1159
1160   return Buffer;
1161 }
1162
1163 /**
1164   Special handler for #DB exception, which will restore the page attributes
1165   (not-present). It should work with #PF handler which will set pages to
1166   'present'.
1167
1168   @param ExceptionType  Exception type.
1169   @param SystemContext  Pointer to EFI_SYSTEM_CONTEXT.
1170
1171 **/
1172 VOID
1173 EFIAPI
1174 DebugExceptionHandler (
1175   IN EFI_EXCEPTION_TYPE   ExceptionType,
1176   IN EFI_SYSTEM_CONTEXT   SystemContext
1177   )
1178 {
1179   UINTN     CpuIndex;
1180   UINTN     PFEntry;
1181   BOOLEAN   IsWpEnabled;
1182
1183   MpInitLibWhoAmI (&CpuIndex);
1184
1185   //
1186   // Clear last PF entries
1187   //
1188   IsWpEnabled = IsReadOnlyPageWriteProtected ();
1189   if (IsWpEnabled) {
1190     DisableReadOnlyPageWriteProtect ();
1191   }
1192
1193   for (PFEntry = 0; PFEntry < mPFEntryCount[CpuIndex]; PFEntry++) {
1194     if (mLastPFEntryPointer[CpuIndex][PFEntry] != NULL) {
1195       *mLastPFEntryPointer[CpuIndex][PFEntry] &= ~(UINT64)IA32_PG_P;
1196     }
1197   }
1198
1199   if (IsWpEnabled) {
1200     EnableReadOnlyPageWriteProtect ();
1201   }
1202
1203   //
1204   // Reset page fault exception count for next page fault.
1205   //
1206   mPFEntryCount[CpuIndex] = 0;
1207
1208   //
1209   // Flush TLB
1210   //
1211   CpuFlushTlb ();
1212
1213   //
1214   // Clear TF in EFLAGS
1215   //
1216   if (mPagingContext.MachineType == IMAGE_FILE_MACHINE_I386) {
1217     SystemContext.SystemContextIa32->Eflags &= (UINT32)~BIT8;
1218   } else {
1219     SystemContext.SystemContextX64->Rflags &= (UINT64)~BIT8;
1220   }
1221 }
1222
1223 /**
1224   Special handler for #PF exception, which will set the pages which caused
1225   #PF to be 'present'. The attribute of those pages should be restored in
1226   the subsequent #DB handler.
1227
1228   @param ExceptionType  Exception type.
1229   @param SystemContext  Pointer to EFI_SYSTEM_CONTEXT.
1230
1231 **/
1232 VOID
1233 EFIAPI
1234 PageFaultExceptionHandler (
1235   IN EFI_EXCEPTION_TYPE   ExceptionType,
1236   IN EFI_SYSTEM_CONTEXT   SystemContext
1237   )
1238 {
1239   EFI_STATUS                      Status;
1240   UINT64                          PFAddress;
1241   PAGE_TABLE_LIB_PAGING_CONTEXT   PagingContext;
1242   PAGE_ATTRIBUTE                  PageAttribute;
1243   UINT64                          Attributes;
1244   UINT64                          *PageEntry;
1245   UINTN                           Index;
1246   UINTN                           CpuIndex;
1247   UINTN                           PageNumber;
1248   BOOLEAN                         NonStopMode;
1249
1250   PFAddress = AsmReadCr2 () & ~EFI_PAGE_MASK;
1251   if (PFAddress < BASE_4KB) {
1252     NonStopMode = NULL_DETECTION_NONSTOP_MODE ? TRUE : FALSE;
1253   } else {
1254     NonStopMode = HEAP_GUARD_NONSTOP_MODE ? TRUE : FALSE;
1255   }
1256
1257   if (NonStopMode) {
1258     MpInitLibWhoAmI (&CpuIndex);
1259     GetCurrentPagingContext (&PagingContext);
1260     //
1261     // Memory operation cross page boundary, like "rep mov" instruction, will
1262     // cause infinite loop between this and Debug Trap handler. We have to make
1263     // sure that current page and the page followed are both in PRESENT state.
1264     //
1265     PageNumber = 2;
1266     while (PageNumber > 0) {
1267       PageEntry = GetPageTableEntry (&PagingContext, PFAddress, &PageAttribute);
1268       ASSERT(PageEntry != NULL);
1269
1270       if (PageEntry != NULL) {
1271         Attributes = GetAttributesFromPageEntry (PageEntry);
1272         if ((Attributes & EFI_MEMORY_RP) != 0) {
1273           Attributes &= ~EFI_MEMORY_RP;
1274           Status = AssignMemoryPageAttributes (&PagingContext, PFAddress,
1275                                                EFI_PAGE_SIZE, Attributes, NULL);
1276           if (!EFI_ERROR(Status)) {
1277             Index = mPFEntryCount[CpuIndex];
1278             //
1279             // Re-retrieve page entry because above calling might update page
1280             // table due to table split.
1281             //
1282             PageEntry = GetPageTableEntry (&PagingContext, PFAddress, &PageAttribute);
1283             mLastPFEntryPointer[CpuIndex][Index++] = PageEntry;
1284             mPFEntryCount[CpuIndex] = Index;
1285           }
1286         }
1287       }
1288
1289       PFAddress += EFI_PAGE_SIZE;
1290       --PageNumber;
1291     }
1292   }
1293
1294   //
1295   // Initialize the serial port before dumping.
1296   //
1297   SerialPortInitialize ();
1298   //
1299   // Display ExceptionType, CPU information and Image information
1300   //
1301   DumpCpuContext (ExceptionType, SystemContext);
1302   if (NonStopMode) {
1303     //
1304     // Set TF in EFLAGS
1305     //
1306     if (mPagingContext.MachineType == IMAGE_FILE_MACHINE_I386) {
1307       SystemContext.SystemContextIa32->Eflags |= (UINT32)BIT8;
1308     } else {
1309       SystemContext.SystemContextX64->Rflags |= (UINT64)BIT8;
1310     }
1311   } else {
1312     CpuDeadLoop ();
1313   }
1314 }
1315
1316 /**
1317   Initialize the Page Table lib.
1318 **/
1319 VOID
1320 InitializePageTableLib (
1321   VOID
1322   )
1323 {
1324   PAGE_TABLE_LIB_PAGING_CONTEXT     CurrentPagingContext;
1325
1326   GetCurrentPagingContext (&CurrentPagingContext);
1327
1328   //
1329   // Reserve memory of page tables for future uses, if paging is enabled.
1330   //
1331   if (CurrentPagingContext.ContextData.X64.PageTableBase != 0 &&
1332       (CurrentPagingContext.ContextData.Ia32.Attributes &
1333        PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE) != 0) {
1334     DisableReadOnlyPageWriteProtect ();
1335     InitializePageTablePool (1);
1336     EnableReadOnlyPageWriteProtect ();
1337   }
1338
1339   if (HEAP_GUARD_NONSTOP_MODE || NULL_DETECTION_NONSTOP_MODE) {
1340     mPFEntryCount = (UINTN *)AllocateZeroPool (sizeof (UINTN) * mNumberOfProcessors);
1341     ASSERT (mPFEntryCount != NULL);
1342
1343     mLastPFEntryPointer = (UINT64 *(*)[MAX_PF_ENTRY_COUNT])
1344                           AllocateZeroPool (sizeof (mLastPFEntryPointer[0]) * mNumberOfProcessors);
1345     ASSERT (mLastPFEntryPointer != NULL);
1346   }
1347
1348   DEBUG ((DEBUG_INFO, "CurrentPagingContext:\n", CurrentPagingContext.MachineType));
1349   DEBUG ((DEBUG_INFO, "  MachineType   - 0x%x\n", CurrentPagingContext.MachineType));
1350   DEBUG ((DEBUG_INFO, "  PageTableBase - 0x%x\n", CurrentPagingContext.ContextData.X64.PageTableBase));
1351   DEBUG ((DEBUG_INFO, "  Attributes    - 0x%x\n", CurrentPagingContext.ContextData.X64.Attributes));
1352
1353   return ;
1354 }
1355