UefiCpuPkg/CpuDxe/CpuPageTable.c

   1 /** @file
   2   Page table management support.
   3
   4   Copyright (c) 2017 - 2019, Intel Corporation. All rights reserved.<BR>
   5   Copyright (c) 2017, AMD Incorporated. All rights reserved.<BR>
   6
   7   SPDX-License-Identifier: BSD-2-Clause-Patent
   8
   9 **/
  10
  11 #include <Base.h>
  12 #include <Uefi.h>
  13 #include <Library/PeCoffGetEntryPointLib.h>
  14 #include <Library/SerialPortLib.h>
  15 #include <Library/SynchronizationLib.h>
  16 #include <Library/PrintLib.h>
  17 #include <Protocol/SmmBase2.h>
  18 #include <Register/Intel/Cpuid.h>
  19 #include <Register/Intel/Msr.h>
  20
  21 #include "CpuDxe.h"
  22 #include "CpuPageTable.h"
  23
  24 ///
  25 /// Page Table Entry
  26 ///
  27 #define IA32_PG_P       BIT0
  28 #define IA32_PG_RW      BIT1
  29 #define IA32_PG_U       BIT2
  30 #define IA32_PG_WT      BIT3
  31 #define IA32_PG_CD      BIT4
  32 #define IA32_PG_A       BIT5
  33 #define IA32_PG_D       BIT6
  34 #define IA32_PG_PS      BIT7
  35 #define IA32_PG_PAT_2M  BIT12
  36 #define IA32_PG_PAT_4K  IA32_PG_PS
  37 #define IA32_PG_PMNT    BIT62
  38 #define IA32_PG_NX      BIT63
  39
  40 #define PAGE_ATTRIBUTE_BITS             (IA32_PG_D | IA32_PG_A | IA32_PG_U | IA32_PG_RW | IA32_PG_P)
  41 #define PAGE_ATTRIBUTE_BITS_POST_SPLIT  (IA32_PG_RW | IA32_PG_P)
  42
  43 //
  44 // Bits 1, 2, 5, 6 are reserved in the IA32 PAE PDPTE
  45 // X64 PAE PDPTE does not have such restriction
  46 //
  47 #define IA32_PAE_PDPTE_ATTRIBUTE_BITS  (IA32_PG_P)
  48
  49 #define PAGE_PROGATE_BITS  (IA32_PG_NX | PAGE_ATTRIBUTE_BITS)
  50
  51 #define PAGING_4K_MASK  0xFFF
  52 #define PAGING_2M_MASK  0x1FFFFF
  53 #define PAGING_1G_MASK  0x3FFFFFFF
  54
  55 #define PAGING_PAE_INDEX_MASK  0x1FF
  56
  57 #define PAGING_4K_ADDRESS_MASK_64  0x000FFFFFFFFFF000ull
  58 #define PAGING_2M_ADDRESS_MASK_64  0x000FFFFFFFE00000ull
  59 #define PAGING_1G_ADDRESS_MASK_64  0x000FFFFFC0000000ull
  60
  61 #define MAX_PF_ENTRY_COUNT        10
  62 #define MAX_DEBUG_MESSAGE_LENGTH  0x100
  63 #define IA32_PF_EC_ID             BIT4
  64
  65 typedef enum {
  66   PageNone,
  67   Page4K,
  68   Page2M,
  69   Page1G,
  70 } PAGE_ATTRIBUTE;
  71
  72 typedef struct {
  73   PAGE_ATTRIBUTE    Attribute;
  74   UINT64            Length;
  75   UINT64            AddressMask;
  76 } PAGE_ATTRIBUTE_TABLE;
  77
  78 typedef enum {
  79   PageActionAssign,
  80   PageActionSet,
  81   PageActionClear,
  82 } PAGE_ACTION;
  83
  84 PAGE_ATTRIBUTE_TABLE  mPageAttributeTable[] = {
  85   { Page4K, SIZE_4KB, PAGING_4K_ADDRESS_MASK_64 },
  86   { Page2M, SIZE_2MB, PAGING_2M_ADDRESS_MASK_64 },
  87   { Page1G, SIZE_1GB, PAGING_1G_ADDRESS_MASK_64 },
  88 };
  89
  90 PAGE_TABLE_POOL                *mPageTablePool    = NULL;
  91 BOOLEAN                        mPageTablePoolLock = FALSE;
  92 PAGE_TABLE_LIB_PAGING_CONTEXT  mPagingContext;
  93 EFI_SMM_BASE2_PROTOCOL         *mSmmBase2 = NULL;
  94
  95 //
  96 // Record the page fault exception count for one instruction execution.
  97 //
  98 UINTN  *mPFEntryCount;
  99 UINT64                    *(*mLastPFEntryPointer)[MAX_PF_ENTRY_COUNT];
 100
 101 /**
 102  Check if current execution environment is in SMM mode or not, via
 103  EFI_SMM_BASE2_PROTOCOL.
 104
 105  This is necessary because of the fact that MdePkg\Library\SmmMemoryAllocationLib
 106  supports to free memory outside SMRAM. The library will call gBS->FreePool() or
 107  gBS->FreePages() and then SetMemorySpaceAttributes interface in turn to change
 108  memory paging attributes during free operation, if some memory related features
 109  are enabled (like Heap Guard).
 110
 111  This means that SetMemorySpaceAttributes() has chance to run in SMM mode. This
 112  will cause incorrect result because SMM mode always loads its own page tables,
 113  which are usually different from DXE. This function can be used to detect such
 114  situation and help to avoid further misoperations.
 115
 116   @retval TRUE    In SMM mode.
 117   @retval FALSE   Not in SMM mode.
 118 **/
 119 BOOLEAN
 120 IsInSmm (
 121   VOID
 122   )
 123 {
 124   BOOLEAN  InSmm;
 125
 126   InSmm = FALSE;
 127   if (mSmmBase2 == NULL) {
 128     gBS->LocateProtocol (&gEfiSmmBase2ProtocolGuid, NULL, (VOID **)&mSmmBase2);
 129   }
 130
 131   if (mSmmBase2 != NULL) {
 132     mSmmBase2->InSmm (mSmmBase2, &InSmm);
 133   }
 134
 135   //
 136   // mSmmBase2->InSmm() can only detect if the caller is running in SMRAM
 137   // or from SMM driver. It cannot tell if the caller is running in SMM mode.
 138   // Check page table base address to guarantee that because SMM mode willl
 139   // load its own page table.
 140   //
 141   return (InSmm &&
 142           mPagingContext.ContextData.X64.PageTableBase != (UINT64)AsmReadCr3 ());
 143 }
 144
 145 /**
 146   Return current paging context.
 147
 148   @param[in,out]  PagingContext     The paging context.
 149 **/
 150 VOID
 151 GetCurrentPagingContext (
 152   IN OUT PAGE_TABLE_LIB_PAGING_CONTEXT  *PagingContext
 153   )
 154 {
 155   UINT32                      RegEax;
 156   CPUID_EXTENDED_CPU_SIG_EDX  RegEdx;
 157   MSR_IA32_EFER_REGISTER      MsrEfer;
 158   IA32_CR4                    Cr4;
 159   IA32_CR0                    Cr0;
 160   UINT32                      *Attributes;
 161   UINTN                       *PageTableBase;
 162
 163   //
 164   // Don't retrieve current paging context from processor if in SMM mode.
 165   //
 166   if (!IsInSmm ()) {
 167     ZeroMem (&mPagingContext, sizeof (mPagingContext));
 168     if (sizeof (UINTN) == sizeof (UINT64)) {
 169       mPagingContext.MachineType = IMAGE_FILE_MACHINE_X64;
 170     } else {
 171       mPagingContext.MachineType = IMAGE_FILE_MACHINE_I386;
 172     }
 173
 174     GetPagingDetails (&mPagingContext.ContextData, &PageTableBase, &Attributes);
 175
 176     Cr0.UintN = AsmReadCr0 ();
 177     Cr4.UintN = AsmReadCr4 ();
 178
 179     if (Cr0.Bits.PG != 0) {
 180       *PageTableBase = (AsmReadCr3 () & PAGING_4K_ADDRESS_MASK_64);
 181     } else {
 182       *PageTableBase = 0;
 183     }
 184
 185     if (Cr0.Bits.WP  != 0) {
 186       *Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_WP_ENABLE;
 187     }
 188
 189     if (Cr4.Bits.PSE != 0) {
 190       *Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PSE;
 191     }
 192
 193     if (Cr4.Bits.PAE != 0) {
 194       *Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE;
 195     }
 196
 197     if (Cr4.Bits.LA57 != 0) {
 198       *Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_5_LEVEL;
 199     }
 200
 201     AsmCpuid (CPUID_EXTENDED_FUNCTION, &RegEax, NULL, NULL, NULL);
 202     if (RegEax >= CPUID_EXTENDED_CPU_SIG) {
 203       AsmCpuid (CPUID_EXTENDED_CPU_SIG, NULL, NULL, NULL, &RegEdx.Uint32);
 204
 205       if (RegEdx.Bits.NX != 0) {
 206         // XD supported
 207         MsrEfer.Uint64 = AsmReadMsr64 (MSR_CORE_IA32_EFER);
 208         if (MsrEfer.Bits.NXE != 0) {
 209           // XD activated
 210           *Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_XD_ACTIVATED;
 211         }
 212       }
 213
 214       if (RegEdx.Bits.Page1GB != 0) {
 215         *Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAGE_1G_SUPPORT;
 216       }
 217     }
 218   }
 219
 220   //
 221   // This can avoid getting SMM paging context if in SMM mode. We cannot assume
 222   // SMM mode shares the same paging context as DXE.
 223   //
 224   CopyMem (PagingContext, &mPagingContext, sizeof (mPagingContext));
 225 }
 226
 227 /**
 228   Return length according to page attributes.
 229
 230   @param[in]  PageAttributes   The page attribute of the page entry.
 231
 232   @return The length of page entry.
 233 **/
 234 UINTN
 235 PageAttributeToLength (
 236   IN PAGE_ATTRIBUTE  PageAttribute
 237   )
 238 {
 239   UINTN  Index;
 240
 241   for (Index = 0; Index < sizeof (mPageAttributeTable)/sizeof (mPageAttributeTable[0]); Index++) {
 242     if (PageAttribute == mPageAttributeTable[Index].Attribute) {
 243       return (UINTN)mPageAttributeTable[Index].Length;
 244     }
 245   }
 246
 247   return 0;
 248 }
 249
 250 /**
 251   Return address mask according to page attributes.
 252
 253   @param[in]  PageAttributes   The page attribute of the page entry.
 254
 255   @return The address mask of page entry.
 256 **/
 257 UINTN
 258 PageAttributeToMask (
 259   IN PAGE_ATTRIBUTE  PageAttribute
 260   )
 261 {
 262   UINTN  Index;
 263
 264   for (Index = 0; Index < sizeof (mPageAttributeTable)/sizeof (mPageAttributeTable[0]); Index++) {
 265     if (PageAttribute == mPageAttributeTable[Index].Attribute) {
 266       return (UINTN)mPageAttributeTable[Index].AddressMask;
 267     }
 268   }
 269
 270   return 0;
 271 }
 272
 273 /**
 274   Return page table entry to match the address.
 275
 276   @param[in]  PagingContext     The paging context.
 277   @param[in]  Address           The address to be checked.
 278   @param[out] PageAttributes    The page attribute of the page entry.
 279
 280   @return The page entry.
 281 **/
 282 VOID *
 283 GetPageTableEntry (
 284   IN  PAGE_TABLE_LIB_PAGING_CONTEXT  *PagingContext,
 285   IN  PHYSICAL_ADDRESS               Address,
 286   OUT PAGE_ATTRIBUTE                 *PageAttribute
 287   )
 288 {
 289   UINTN   Index1;
 290   UINTN   Index2;
 291   UINTN   Index3;
 292   UINTN   Index4;
 293   UINTN   Index5;
 294   UINT64  *L1PageTable;
 295   UINT64  *L2PageTable;
 296   UINT64  *L3PageTable;
 297   UINT64  *L4PageTable;
 298   UINT64  *L5PageTable;
 299   UINT64  AddressEncMask;
 300
 301   ASSERT (PagingContext != NULL);
 302
 303   Index5 = ((UINTN)RShiftU64 (Address, 48)) & PAGING_PAE_INDEX_MASK;
 304   Index4 = ((UINTN)RShiftU64 (Address, 39)) & PAGING_PAE_INDEX_MASK;
 305   Index3 = ((UINTN)Address >> 30) & PAGING_PAE_INDEX_MASK;
 306   Index2 = ((UINTN)Address >> 21) & PAGING_PAE_INDEX_MASK;
 307   Index1 = ((UINTN)Address >> 12) & PAGING_PAE_INDEX_MASK;
 308
 309   // Make sure AddressEncMask is contained to smallest supported address field.
 310   //
 311   AddressEncMask = PcdGet64 (PcdPteMemoryEncryptionAddressOrMask) & PAGING_1G_ADDRESS_MASK_64;
 312   if (AddressEncMask == 0) {
 313     AddressEncMask = PcdGet64 (PcdTdxSharedBitMask) & PAGING_1G_ADDRESS_MASK_64;
 314   }
 315
 316   if (PagingContext->MachineType == IMAGE_FILE_MACHINE_X64) {
 317     if ((PagingContext->ContextData.X64.Attributes & PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_5_LEVEL) != 0) {
 318       L5PageTable = (UINT64 *)(UINTN)PagingContext->ContextData.X64.PageTableBase;
 319       if (L5PageTable[Index5] == 0) {
 320         *PageAttribute = PageNone;
 321         return NULL;
 322       }
 323
 324       L4PageTable = (UINT64 *)(UINTN)(L5PageTable[Index5] & ~AddressEncMask & PAGING_4K_ADDRESS_MASK_64);
 325     } else {
 326       L4PageTable = (UINT64 *)(UINTN)PagingContext->ContextData.X64.PageTableBase;
 327     }
 328
 329     if (L4PageTable[Index4] == 0) {
 330       *PageAttribute = PageNone;
 331       return NULL;
 332     }
 333
 334     L3PageTable = (UINT64 *)(UINTN)(L4PageTable[Index4] & ~AddressEncMask & PAGING_4K_ADDRESS_MASK_64);
 335   } else {
 336     ASSERT ((PagingContext->ContextData.Ia32.Attributes & PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE) != 0);
 337     L3PageTable = (UINT64 *)(UINTN)PagingContext->ContextData.Ia32.PageTableBase;
 338   }
 339
 340   if (L3PageTable[Index3] == 0) {
 341     *PageAttribute = PageNone;
 342     return NULL;
 343   }
 344
 345   if ((L3PageTable[Index3] & IA32_PG_PS) != 0) {
 346     // 1G
 347     *PageAttribute = Page1G;
 348     return &L3PageTable[Index3];
 349   }
 350
 351   L2PageTable = (UINT64 *)(UINTN)(L3PageTable[Index3] & ~AddressEncMask & PAGING_4K_ADDRESS_MASK_64);
 352   if (L2PageTable[Index2] == 0) {
 353     *PageAttribute = PageNone;
 354     return NULL;
 355   }
 356
 357   if ((L2PageTable[Index2] & IA32_PG_PS) != 0) {
 358     // 2M
 359     *PageAttribute = Page2M;
 360     return &L2PageTable[Index2];
 361   }
 362
 363   // 4k
 364   L1PageTable = (UINT64 *)(UINTN)(L2PageTable[Index2] & ~AddressEncMask & PAGING_4K_ADDRESS_MASK_64);
 365   if ((L1PageTable[Index1] == 0) && (Address != 0)) {
 366     *PageAttribute = PageNone;
 367     return NULL;
 368   }
 369
 370   *PageAttribute = Page4K;
 371   return &L1PageTable[Index1];
 372 }
 373
 374 /**
 375   Return memory attributes of page entry.
 376
 377   @param[in]  PageEntry        The page entry.
 378
 379   @return Memory attributes of page entry.
 380 **/
 381 UINT64
 382 GetAttributesFromPageEntry (
 383   IN  UINT64  *PageEntry
 384   )
 385 {
 386   UINT64  Attributes;
 387
 388   Attributes = 0;
 389   if ((*PageEntry & IA32_PG_P) == 0) {
 390     Attributes |= EFI_MEMORY_RP;
 391   }
 392
 393   if ((*PageEntry & IA32_PG_RW) == 0) {
 394     Attributes |= EFI_MEMORY_RO;
 395   }
 396
 397   if ((*PageEntry & IA32_PG_NX) != 0) {
 398     Attributes |= EFI_MEMORY_XP;
 399   }
 400
 401   return Attributes;
 402 }
 403
 404 /**
 405   Modify memory attributes of page entry.
 406
 407   @param[in]  PagingContext    The paging context.
 408   @param[in]  PageEntry        The page entry.
 409   @param[in]  Attributes       The bit mask of attributes to modify for the memory region.
 410   @param[in]  PageAction       The page action.
 411   @param[out] IsModified       TRUE means page table modified. FALSE means page table not modified.
 412 **/
 413 VOID
 414 ConvertPageEntryAttribute (
 415   IN  PAGE_TABLE_LIB_PAGING_CONTEXT  *PagingContext,
 416   IN  UINT64                         *PageEntry,
 417   IN  UINT64                         Attributes,
 418   IN  PAGE_ACTION                    PageAction,
 419   OUT BOOLEAN                        *IsModified
 420   )
 421 {
 422   UINT64  CurrentPageEntry;
 423   UINT64  NewPageEntry;
 424   UINT32  *PageAttributes;
 425
 426   CurrentPageEntry = *PageEntry;
 427   NewPageEntry     = CurrentPageEntry;
 428   if ((Attributes & EFI_MEMORY_RP) != 0) {
 429     switch (PageAction) {
 430       case PageActionAssign:
 431       case PageActionSet:
 432         NewPageEntry &= ~(UINT64)IA32_PG_P;
 433         break;
 434       case PageActionClear:
 435         NewPageEntry |= IA32_PG_P;
 436         break;
 437     }
 438   } else {
 439     switch (PageAction) {
 440       case PageActionAssign:
 441         NewPageEntry |= IA32_PG_P;
 442         break;
 443       case PageActionSet:
 444       case PageActionClear:
 445         break;
 446     }
 447   }
 448
 449   if ((Attributes & EFI_MEMORY_RO) != 0) {
 450     switch (PageAction) {
 451       case PageActionAssign:
 452       case PageActionSet:
 453         NewPageEntry &= ~(UINT64)IA32_PG_RW;
 454         break;
 455       case PageActionClear:
 456         NewPageEntry |= IA32_PG_RW;
 457         break;
 458     }
 459   } else {
 460     switch (PageAction) {
 461       case PageActionAssign:
 462         NewPageEntry |= IA32_PG_RW;
 463         break;
 464       case PageActionSet:
 465       case PageActionClear:
 466         break;
 467     }
 468   }
 469
 470   GetPagingDetails (&PagingContext->ContextData, NULL, &PageAttributes);
 471
 472   if ((*PageAttributes & PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_XD_ACTIVATED) != 0) {
 473     if ((Attributes & EFI_MEMORY_XP) != 0) {
 474       switch (PageAction) {
 475         case PageActionAssign:
 476         case PageActionSet:
 477           NewPageEntry |= IA32_PG_NX;
 478           break;
 479         case PageActionClear:
 480           NewPageEntry &= ~IA32_PG_NX;
 481           break;
 482       }
 483     } else {
 484       switch (PageAction) {
 485         case PageActionAssign:
 486           NewPageEntry &= ~IA32_PG_NX;
 487           break;
 488         case PageActionSet:
 489         case PageActionClear:
 490           break;
 491       }
 492     }
 493   }
 494
 495   *PageEntry = NewPageEntry;
 496   if (CurrentPageEntry != NewPageEntry) {
 497     *IsModified = TRUE;
 498     DEBUG ((DEBUG_VERBOSE, "ConvertPageEntryAttribute 0x%lx", CurrentPageEntry));
 499     DEBUG ((DEBUG_VERBOSE, "->0x%lx\n", NewPageEntry));
 500   } else {
 501     *IsModified = FALSE;
 502   }
 503 }
 504
 505 /**
 506   This function returns if there is need to split page entry.
 507
 508   @param[in]  BaseAddress      The base address to be checked.
 509   @param[in]  Length           The length to be checked.
 510   @param[in]  PageEntry        The page entry to be checked.
 511   @param[in]  PageAttribute    The page attribute of the page entry.
 512
 513   @retval SplitAttributes on if there is need to split page entry.
 514 **/
 515 PAGE_ATTRIBUTE
 516 NeedSplitPage (
 517   IN  PHYSICAL_ADDRESS  BaseAddress,
 518   IN  UINT64            Length,
 519   IN  UINT64            *PageEntry,
 520   IN  PAGE_ATTRIBUTE    PageAttribute
 521   )
 522 {
 523   UINT64  PageEntryLength;
 524
 525   PageEntryLength = PageAttributeToLength (PageAttribute);
 526
 527   if (((BaseAddress & (PageEntryLength - 1)) == 0) && (Length >= PageEntryLength)) {
 528     return PageNone;
 529   }
 530
 531   if (((BaseAddress & PAGING_2M_MASK) != 0) || (Length < SIZE_2MB)) {
 532     return Page4K;
 533   }
 534
 535   return Page2M;
 536 }
 537
 538 /**
 539   This function splits one page entry to small page entries.
 540
 541   @param[in]  PageEntry         The page entry to be splitted.
 542   @param[in]  PageAttribute     The page attribute of the page entry.
 543   @param[in]  SplitAttribute    How to split the page entry.
 544   @param[in]  AllocatePagesFunc If page split is needed, this function is used to allocate more pages.
 545
 546   @retval RETURN_SUCCESS            The page entry is splitted.
 547   @retval RETURN_UNSUPPORTED        The page entry does not support to be splitted.
 548   @retval RETURN_OUT_OF_RESOURCES   No resource to split page entry.
 549 **/
 550 RETURN_STATUS
 551 SplitPage (
 552   IN  UINT64                         *PageEntry,
 553   IN  PAGE_ATTRIBUTE                 PageAttribute,
 554   IN  PAGE_ATTRIBUTE                 SplitAttribute,
 555   IN  PAGE_TABLE_LIB_ALLOCATE_PAGES  AllocatePagesFunc
 556   )
 557 {
 558   UINT64  BaseAddress;
 559   UINT64  *NewPageEntry;
 560   UINTN   Index;
 561   UINT64  AddressEncMask;
 562
 563   ASSERT (PageAttribute == Page2M || PageAttribute == Page1G);
 564
 565   ASSERT (AllocatePagesFunc != NULL);
 566
 567   // Make sure AddressEncMask is contained to smallest supported address field.
 568   //
 569   AddressEncMask = PcdGet64 (PcdPteMemoryEncryptionAddressOrMask) & PAGING_1G_ADDRESS_MASK_64;
 570
 571   if (PageAttribute == Page2M) {
 572     //
 573     // Split 2M to 4K
 574     //
 575     ASSERT (SplitAttribute == Page4K);
 576     if (SplitAttribute == Page4K) {
 577       NewPageEntry = AllocatePagesFunc (1);
 578       DEBUG ((DEBUG_VERBOSE, "Split - 0x%x\n", NewPageEntry));
 579       if (NewPageEntry == NULL) {
 580         return RETURN_OUT_OF_RESOURCES;
 581       }
 582
 583       BaseAddress = *PageEntry & ~AddressEncMask & PAGING_2M_ADDRESS_MASK_64;
 584       for (Index = 0; Index < SIZE_4KB / sizeof (UINT64); Index++) {
 585         NewPageEntry[Index] = (BaseAddress + SIZE_4KB * Index) | AddressEncMask | ((*PageEntry) & PAGE_PROGATE_BITS);
 586       }
 587
 588       (*PageEntry) = (UINT64)(UINTN)NewPageEntry | AddressEncMask | PAGE_ATTRIBUTE_BITS_POST_SPLIT;
 589       return RETURN_SUCCESS;
 590     } else {
 591       return RETURN_UNSUPPORTED;
 592     }
 593   } else if (PageAttribute == Page1G) {
 594     //
 595     // Split 1G to 2M
 596     // No need support 1G->4K directly, we should use 1G->2M, then 2M->4K to get more compact page table.
 597     //
 598     ASSERT (SplitAttribute == Page2M || SplitAttribute == Page4K);
 599     if (((SplitAttribute == Page2M) || (SplitAttribute == Page4K))) {
 600       NewPageEntry = AllocatePagesFunc (1);
 601       DEBUG ((DEBUG_VERBOSE, "Split - 0x%x\n", NewPageEntry));
 602       if (NewPageEntry == NULL) {
 603         return RETURN_OUT_OF_RESOURCES;
 604       }
 605
 606       BaseAddress = *PageEntry & ~AddressEncMask  & PAGING_1G_ADDRESS_MASK_64;
 607       for (Index = 0; Index < SIZE_4KB / sizeof (UINT64); Index++) {
 608         NewPageEntry[Index] = (BaseAddress + SIZE_2MB * Index) | AddressEncMask | IA32_PG_PS | ((*PageEntry) & PAGE_PROGATE_BITS);
 609       }
 610
 611       (*PageEntry) = (UINT64)(UINTN)NewPageEntry | AddressEncMask | PAGE_ATTRIBUTE_BITS_POST_SPLIT;
 612       return RETURN_SUCCESS;
 613     } else {
 614       return RETURN_UNSUPPORTED;
 615     }
 616   } else {
 617     return RETURN_UNSUPPORTED;
 618   }
 619 }
 620
 621 /**
 622  Check the WP status in CR0 register. This bit is used to lock or unlock write
 623  access to pages marked as read-only.
 624
 625   @retval TRUE    Write protection is enabled.
 626   @retval FALSE   Write protection is disabled.
 627 **/
 628 BOOLEAN
 629 IsReadOnlyPageWriteProtected (
 630   VOID
 631   )
 632 {
 633   IA32_CR0  Cr0;
 634
 635   //
 636   // To avoid unforseen consequences, don't touch paging settings in SMM mode
 637   // in this driver.
 638   //
 639   if (!IsInSmm ()) {
 640     Cr0.UintN = AsmReadCr0 ();
 641     return (BOOLEAN)(Cr0.Bits.WP != 0);
 642   }
 643
 644   return FALSE;
 645 }
 646
 647 /**
 648  Disable Write Protect on pages marked as read-only.
 649 **/
 650 VOID
 651 DisableReadOnlyPageWriteProtect (
 652   VOID
 653   )
 654 {
 655   IA32_CR0  Cr0;
 656
 657   //
 658   // To avoid unforseen consequences, don't touch paging settings in SMM mode
 659   // in this driver.
 660   //
 661   if (!IsInSmm ()) {
 662     Cr0.UintN   = AsmReadCr0 ();
 663     Cr0.Bits.WP = 0;
 664     AsmWriteCr0 (Cr0.UintN);
 665   }
 666 }
 667
 668 /**
 669  Enable Write Protect on pages marked as read-only.
 670 **/
 671 VOID
 672 EnableReadOnlyPageWriteProtect (
 673   VOID
 674   )
 675 {
 676   IA32_CR0  Cr0;
 677
 678   //
 679   // To avoid unforseen consequences, don't touch paging settings in SMM mode
 680   // in this driver.
 681   //
 682   if (!IsInSmm ()) {
 683     Cr0.UintN   = AsmReadCr0 ();
 684     Cr0.Bits.WP = 1;
 685     AsmWriteCr0 (Cr0.UintN);
 686   }
 687 }
 688
 689 /**
 690   This function modifies the page attributes for the memory region specified by BaseAddress and
 691   Length from their current attributes to the attributes specified by Attributes.
 692
 693   Caller should make sure BaseAddress and Length is at page boundary.
 694
 695   @param[in]  PagingContext     The paging context. NULL means get page table from current CPU context.
 696   @param[in]  BaseAddress       The physical address that is the start address of a memory region.
 697   @param[in]  Length            The size in bytes of the memory region.
 698   @param[in]  Attributes        The bit mask of attributes to modify for the memory region.
 699   @param[in]  PageAction        The page action.
 700   @param[in]  AllocatePagesFunc If page split is needed, this function is used to allocate more pages.
 701                                 NULL mean page split is unsupported.
 702   @param[out] IsSplitted        TRUE means page table splitted. FALSE means page table not splitted.
 703   @param[out] IsModified        TRUE means page table modified. FALSE means page table not modified.
 704
 705   @retval RETURN_SUCCESS           The attributes were modified for the memory region.
 706   @retval RETURN_ACCESS_DENIED     The attributes for the memory resource range specified by
 707                                    BaseAddress and Length cannot be modified.
 708   @retval RETURN_INVALID_PARAMETER Length is zero.
 709                                    Attributes specified an illegal combination of attributes that
 710                                    cannot be set together.
 711   @retval RETURN_OUT_OF_RESOURCES  There are not enough system resources to modify the attributes of
 712                                    the memory resource range.
 713   @retval RETURN_UNSUPPORTED       The processor does not support one or more bytes of the memory
 714                                    resource range specified by BaseAddress and Length.
 715                                    The bit mask of attributes is not support for the memory resource
 716                                    range specified by BaseAddress and Length.
 717 **/
 718 RETURN_STATUS
 719 ConvertMemoryPageAttributes (
 720   IN  PAGE_TABLE_LIB_PAGING_CONTEXT  *PagingContext OPTIONAL,
 721   IN  PHYSICAL_ADDRESS               BaseAddress,
 722   IN  UINT64                         Length,
 723   IN  UINT64                         Attributes,
 724   IN  PAGE_ACTION                    PageAction,
 725   IN  PAGE_TABLE_LIB_ALLOCATE_PAGES  AllocatePagesFunc OPTIONAL,
 726   OUT BOOLEAN                        *IsSplitted   OPTIONAL,
 727   OUT BOOLEAN                        *IsModified   OPTIONAL
 728   )
 729 {
 730   PAGE_TABLE_LIB_PAGING_CONTEXT  CurrentPagingContext;
 731   UINT64                         *PageEntry;
 732   PAGE_ATTRIBUTE                 PageAttribute;
 733   UINTN                          PageEntryLength;
 734   PAGE_ATTRIBUTE                 SplitAttribute;
 735   RETURN_STATUS                  Status;
 736   BOOLEAN                        IsEntryModified;
 737   BOOLEAN                        IsWpEnabled;
 738
 739   if ((BaseAddress & (SIZE_4KB - 1)) != 0) {
 740     DEBUG ((DEBUG_ERROR, "BaseAddress(0x%lx) is not aligned!\n", BaseAddress));
 741     return EFI_UNSUPPORTED;
 742   }
 743
 744   if ((Length & (SIZE_4KB - 1)) != 0) {
 745     DEBUG ((DEBUG_ERROR, "Length(0x%lx) is not aligned!\n", Length));
 746     return EFI_UNSUPPORTED;
 747   }
 748
 749   if (Length == 0) {
 750     DEBUG ((DEBUG_ERROR, "Length is 0!\n"));
 751     return RETURN_INVALID_PARAMETER;
 752   }
 753
 754   if ((Attributes & ~EFI_MEMORY_ATTRIBUTE_MASK) != 0) {
 755     DEBUG ((DEBUG_ERROR, "Attributes(0x%lx) has unsupported bit\n", Attributes));
 756     return EFI_UNSUPPORTED;
 757   }
 758
 759   if (PagingContext == NULL) {
 760     GetCurrentPagingContext (&CurrentPagingContext);
 761   } else {
 762     CopyMem (&CurrentPagingContext, PagingContext, sizeof (CurrentPagingContext));
 763   }
 764
 765   switch (CurrentPagingContext.MachineType) {
 766     case IMAGE_FILE_MACHINE_I386:
 767       if (CurrentPagingContext.ContextData.Ia32.PageTableBase == 0) {
 768         if (Attributes == 0) {
 769           return EFI_SUCCESS;
 770         } else {
 771           DEBUG ((DEBUG_ERROR, "PageTable is 0!\n"));
 772           return EFI_UNSUPPORTED;
 773         }
 774       }
 775
 776       if ((CurrentPagingContext.ContextData.Ia32.Attributes & PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE) == 0) {
 777         DEBUG ((DEBUG_ERROR, "Non-PAE Paging!\n"));
 778         return EFI_UNSUPPORTED;
 779       }
 780
 781       if ((BaseAddress + Length) > BASE_4GB) {
 782         DEBUG ((DEBUG_ERROR, "Beyond 4GB memory in 32-bit mode!\n"));
 783         return EFI_UNSUPPORTED;
 784       }
 785
 786       break;
 787     case IMAGE_FILE_MACHINE_X64:
 788       ASSERT (CurrentPagingContext.ContextData.X64.PageTableBase != 0);
 789       break;
 790     default:
 791       ASSERT (FALSE);
 792       return EFI_UNSUPPORTED;
 793       break;
 794   }
 795
 796   //  DEBUG ((DEBUG_ERROR, "ConvertMemoryPageAttributes(%x) - %016lx, %016lx, %02lx\n", IsSet, BaseAddress, Length, Attributes));
 797
 798   if (IsSplitted != NULL) {
 799     *IsSplitted = FALSE;
 800   }
 801
 802   if (IsModified != NULL) {
 803     *IsModified = FALSE;
 804   }
 805
 806   if (AllocatePagesFunc == NULL) {
 807     AllocatePagesFunc = AllocatePageTableMemory;
 808   }
 809
 810   //
 811   // Make sure that the page table is changeable.
 812   //
 813   IsWpEnabled = IsReadOnlyPageWriteProtected ();
 814   if (IsWpEnabled) {
 815     DisableReadOnlyPageWriteProtect ();
 816   }
 817
 818   //
 819   // Below logic is to check 2M/4K page to make sure we do not waste memory.
 820   //
 821   Status = EFI_SUCCESS;
 822   while (Length != 0) {
 823     PageEntry = GetPageTableEntry (&CurrentPagingContext, BaseAddress, &PageAttribute);
 824     if (PageEntry == NULL) {
 825       Status = RETURN_UNSUPPORTED;
 826       goto Done;
 827     }
 828
 829     PageEntryLength = PageAttributeToLength (PageAttribute);
 830     SplitAttribute  = NeedSplitPage (BaseAddress, Length, PageEntry, PageAttribute);
 831     if (SplitAttribute == PageNone) {
 832       ConvertPageEntryAttribute (&CurrentPagingContext, PageEntry, Attributes, PageAction, &IsEntryModified);
 833       if (IsEntryModified) {
 834         if (IsModified != NULL) {
 835           *IsModified = TRUE;
 836         }
 837       }
 838
 839       //
 840       // Convert success, move to next
 841       //
 842       BaseAddress += PageEntryLength;
 843       Length      -= PageEntryLength;
 844     } else {
 845       if (AllocatePagesFunc == NULL) {
 846         Status = RETURN_UNSUPPORTED;
 847         goto Done;
 848       }
 849
 850       Status = SplitPage (PageEntry, PageAttribute, SplitAttribute, AllocatePagesFunc);
 851       if (RETURN_ERROR (Status)) {
 852         Status = RETURN_UNSUPPORTED;
 853         goto Done;
 854       }
 855
 856       if (IsSplitted != NULL) {
 857         *IsSplitted = TRUE;
 858       }
 859
 860       if (IsModified != NULL) {
 861         *IsModified = TRUE;
 862       }
 863
 864       //
 865       // Just split current page
 866       // Convert success in next around
 867       //
 868     }
 869   }
 870
 871 Done:
 872   //
 873   // Restore page table write protection, if any.
 874   //
 875   if (IsWpEnabled) {
 876     EnableReadOnlyPageWriteProtect ();
 877   }
 878
 879   return Status;
 880 }
 881
 882 /**
 883   This function assigns the page attributes for the memory region specified by BaseAddress and
 884   Length from their current attributes to the attributes specified by Attributes.
 885
 886   Caller should make sure BaseAddress and Length is at page boundary.
 887
 888   Caller need guarantee the TPL <= TPL_NOTIFY, if there is split page request.
 889
 890   @param[in]  PagingContext     The paging context. NULL means get page table from current CPU context.
 891   @param[in]  BaseAddress       The physical address that is the start address of a memory region.
 892   @param[in]  Length            The size in bytes of the memory region.
 893   @param[in]  Attributes        The bit mask of attributes to set for the memory region.
 894   @param[in]  AllocatePagesFunc If page split is needed, this function is used to allocate more pages.
 895                                 NULL mean page split is unsupported.
 896
 897   @retval RETURN_SUCCESS           The attributes were cleared for the memory region.
 898   @retval RETURN_ACCESS_DENIED     The attributes for the memory resource range specified by
 899                                    BaseAddress and Length cannot be modified.
 900   @retval RETURN_INVALID_PARAMETER Length is zero.
 901                                    Attributes specified an illegal combination of attributes that
 902                                    cannot be set together.
 903   @retval RETURN_OUT_OF_RESOURCES  There are not enough system resources to modify the attributes of
 904                                    the memory resource range.
 905   @retval RETURN_UNSUPPORTED       The processor does not support one or more bytes of the memory
 906                                    resource range specified by BaseAddress and Length.
 907                                    The bit mask of attributes is not support for the memory resource
 908                                    range specified by BaseAddress and Length.
 909 **/
 910 RETURN_STATUS
 911 EFIAPI
 912 AssignMemoryPageAttributes (
 913   IN  PAGE_TABLE_LIB_PAGING_CONTEXT  *PagingContext OPTIONAL,
 914   IN  PHYSICAL_ADDRESS               BaseAddress,
 915   IN  UINT64                         Length,
 916   IN  UINT64                         Attributes,
 917   IN  PAGE_TABLE_LIB_ALLOCATE_PAGES  AllocatePagesFunc OPTIONAL
 918   )
 919 {
 920   RETURN_STATUS  Status;
 921   BOOLEAN        IsModified;
 922   BOOLEAN        IsSplitted;
 923
 924   //  DEBUG((DEBUG_INFO, "AssignMemoryPageAttributes: 0x%lx - 0x%lx (0x%lx)\n", BaseAddress, Length, Attributes));
 925   Status = ConvertMemoryPageAttributes (PagingContext, BaseAddress, Length, Attributes, PageActionAssign, AllocatePagesFunc, &IsSplitted, &IsModified);
 926   if (!EFI_ERROR (Status)) {
 927     if ((PagingContext == NULL) && IsModified) {
 928       //
 929       // Flush TLB as last step.
 930       //
 931       // Note: Since APs will always init CR3 register in HLT loop mode or do
 932       // TLB flush in MWAIT loop mode, there's no need to flush TLB for them
 933       // here.
 934       //
 935       CpuFlushTlb ();
 936     }
 937   }
 938
 939   return Status;
 940 }
 941
 942 /**
 943  Check if Execute Disable feature is enabled or not.
 944 **/
 945 BOOLEAN
 946 IsExecuteDisableEnabled (
 947   VOID
 948   )
 949 {
 950   MSR_CORE_IA32_EFER_REGISTER  MsrEfer;
 951
 952   MsrEfer.Uint64 = AsmReadMsr64 (MSR_IA32_EFER);
 953   return (MsrEfer.Bits.NXE == 1);
 954 }
 955
 956 /**
 957   Update GCD memory space attributes according to current page table setup.
 958 **/
 959 VOID
 960 RefreshGcdMemoryAttributesFromPaging (
 961   VOID
 962   )
 963 {
 964   EFI_STATUS                       Status;
 965   UINTN                            NumberOfDescriptors;
 966   EFI_GCD_MEMORY_SPACE_DESCRIPTOR  *MemorySpaceMap;
 967   PAGE_TABLE_LIB_PAGING_CONTEXT    PagingContext;
 968   PAGE_ATTRIBUTE                   PageAttribute;
 969   UINT64                           *PageEntry;
 970   UINT64                           PageLength;
 971   UINT64                           MemorySpaceLength;
 972   UINT64                           Length;
 973   UINT64                           BaseAddress;
 974   UINT64                           PageStartAddress;
 975   UINT64                           Attributes;
 976   UINT64                           Capabilities;
 977   UINT64                           NewAttributes;
 978   UINTN                            Index;
 979
 980   //
 981   // Assuming that memory space map returned is sorted already; otherwise sort
 982   // them in the order of lowest address to highest address.
 983   //
 984   Status = gDS->GetMemorySpaceMap (&NumberOfDescriptors, &MemorySpaceMap);
 985   ASSERT_EFI_ERROR (Status);
 986
 987   GetCurrentPagingContext (&PagingContext);
 988
 989   Attributes    = 0;
 990   NewAttributes = 0;
 991   BaseAddress   = 0;
 992   PageLength    = 0;
 993
 994   if (IsExecuteDisableEnabled ()) {
 995     Capabilities = EFI_MEMORY_RO | EFI_MEMORY_RP | EFI_MEMORY_XP;
 996   } else {
 997     Capabilities = EFI_MEMORY_RO | EFI_MEMORY_RP;
 998   }
 999
1000   for (Index = 0; Index < NumberOfDescriptors; Index++) {
1001     if (MemorySpaceMap[Index].GcdMemoryType == EfiGcdMemoryTypeNonExistent) {
1002       continue;
1003     }
1004
1005     //
1006     // Sync the actual paging related capabilities back to GCD service first.
1007     // As a side effect (good one), this can also help to avoid unnecessary
1008     // memory map entries due to the different capabilities of the same type
1009     // memory, such as multiple RT_CODE and RT_DATA entries in memory map,
1010     // which could cause boot failure of some old Linux distro (before v4.3).
1011     //
1012     Status = gDS->SetMemorySpaceCapabilities (
1013                     MemorySpaceMap[Index].BaseAddress,
1014                     MemorySpaceMap[Index].Length,
1015                     MemorySpaceMap[Index].Capabilities | Capabilities
1016                     );
1017     if (EFI_ERROR (Status)) {
1018       //
1019       // If we cannot update the capabilities, we cannot update its
1020       // attributes either. So just simply skip current block of memory.
1021       //
1022       DEBUG ((
1023         DEBUG_WARN,
1024         "Failed to update capability: [%lu] %016lx - %016lx (%016lx -> %016lx)\r\n",
1025         (UINT64)Index,
1026         MemorySpaceMap[Index].BaseAddress,
1027         MemorySpaceMap[Index].BaseAddress + MemorySpaceMap[Index].Length - 1,
1028         MemorySpaceMap[Index].Capabilities,
1029         MemorySpaceMap[Index].Capabilities | Capabilities
1030         ));
1031       continue;
1032     }
1033
1034     if (MemorySpaceMap[Index].BaseAddress >= (BaseAddress + PageLength)) {
1035       //
1036       // Current memory space starts at a new page. Resetting PageLength will
1037       // trigger a retrieval of page attributes at new address.
1038       //
1039       PageLength = 0;
1040     } else {
1041       //
1042       // In case current memory space is not adjacent to last one
1043       //
1044       PageLength -= (MemorySpaceMap[Index].BaseAddress - BaseAddress);
1045     }
1046
1047     //
1048     // Sync actual page attributes to GCD
1049     //
1050     BaseAddress       = MemorySpaceMap[Index].BaseAddress;
1051     MemorySpaceLength = MemorySpaceMap[Index].Length;
1052     while (MemorySpaceLength > 0) {
1053       if (PageLength == 0) {
1054         PageEntry = GetPageTableEntry (&PagingContext, BaseAddress, &PageAttribute);
1055         if (PageEntry == NULL) {
1056           break;
1057         }
1058
1059         //
1060         // Note current memory space might start in the middle of a page
1061         //
1062         PageStartAddress = (*PageEntry) & (UINT64)PageAttributeToMask (PageAttribute);
1063         PageLength       = PageAttributeToLength (PageAttribute) - (BaseAddress - PageStartAddress);
1064         Attributes       = GetAttributesFromPageEntry (PageEntry);
1065       }
1066
1067       Length = MIN (PageLength, MemorySpaceLength);
1068       if (Attributes != (MemorySpaceMap[Index].Attributes &
1069                          EFI_MEMORY_ATTRIBUTE_MASK))
1070       {
1071         NewAttributes = (MemorySpaceMap[Index].Attributes &
1072                          ~EFI_MEMORY_ATTRIBUTE_MASK) | Attributes;
1073         Status = gDS->SetMemorySpaceAttributes (
1074                         BaseAddress,
1075                         Length,
1076                         NewAttributes
1077                         );
1078         ASSERT_EFI_ERROR (Status);
1079         DEBUG ((
1080           DEBUG_VERBOSE,
1081           "Updated memory space attribute: [%lu] %016lx - %016lx (%016lx -> %016lx)\r\n",
1082           (UINT64)Index,
1083           BaseAddress,
1084           BaseAddress + Length - 1,
1085           MemorySpaceMap[Index].Attributes,
1086           NewAttributes
1087           ));
1088       }
1089
1090       PageLength        -= Length;
1091       MemorySpaceLength -= Length;
1092       BaseAddress       += Length;
1093     }
1094   }
1095
1096   FreePool (MemorySpaceMap);
1097 }
1098
1099 /**
1100   Initialize a buffer pool for page table use only.
1101
1102   To reduce the potential split operation on page table, the pages reserved for
1103   page table should be allocated in the times of PAGE_TABLE_POOL_UNIT_PAGES and
1104   at the boundary of PAGE_TABLE_POOL_ALIGNMENT. So the page pool is always
1105   initialized with number of pages greater than or equal to the given PoolPages.
1106
1107   Once the pages in the pool are used up, this method should be called again to
1108   reserve at least another PAGE_TABLE_POOL_UNIT_PAGES. Usually this won't happen
1109   often in practice.
1110
1111   @param[in] PoolPages      The least page number of the pool to be created.
1112
1113   @retval TRUE    The pool is initialized successfully.
1114   @retval FALSE   The memory is out of resource.
1115 **/
1116 BOOLEAN
1117 InitializePageTablePool (
1118   IN  UINTN  PoolPages
1119   )
1120 {
1121   VOID     *Buffer;
1122   BOOLEAN  IsModified;
1123
1124   //
1125   // Do not allow re-entrance.
1126   //
1127   if (mPageTablePoolLock) {
1128     return FALSE;
1129   }
1130
1131   mPageTablePoolLock = TRUE;
1132   IsModified         = FALSE;
1133
1134   //
1135   // Always reserve at least PAGE_TABLE_POOL_UNIT_PAGES, including one page for
1136   // header.
1137   //
1138   PoolPages += 1;   // Add one page for header.
1139   PoolPages  = ((PoolPages - 1) / PAGE_TABLE_POOL_UNIT_PAGES + 1) *
1140                PAGE_TABLE_POOL_UNIT_PAGES;
1141   Buffer = AllocateAlignedPages (PoolPages, PAGE_TABLE_POOL_ALIGNMENT);
1142   if (Buffer == NULL) {
1143     DEBUG ((DEBUG_ERROR, "ERROR: Out of aligned pages\r\n"));
1144     goto Done;
1145   }
1146
1147   DEBUG ((
1148     DEBUG_INFO,
1149     "Paging: added %lu pages to page table pool\r\n",
1150     (UINT64)PoolPages
1151     ));
1152
1153   //
1154   // Link all pools into a list for easier track later.
1155   //
1156   if (mPageTablePool == NULL) {
1157     mPageTablePool           = Buffer;
1158     mPageTablePool->NextPool = mPageTablePool;
1159   } else {
1160     ((PAGE_TABLE_POOL *)Buffer)->NextPool = mPageTablePool->NextPool;
1161     mPageTablePool->NextPool              = Buffer;
1162     mPageTablePool                        = Buffer;
1163   }
1164
1165   //
1166   // Reserve one page for pool header.
1167   //
1168   mPageTablePool->FreePages = PoolPages - 1;
1169   mPageTablePool->Offset    = EFI_PAGES_TO_SIZE (1);
1170
1171   //
1172   // Mark the whole pool pages as read-only.
1173   //
1174   ConvertMemoryPageAttributes (
1175     NULL,
1176     (PHYSICAL_ADDRESS)(UINTN)Buffer,
1177     EFI_PAGES_TO_SIZE (PoolPages),
1178     EFI_MEMORY_RO,
1179     PageActionSet,
1180     AllocatePageTableMemory,
1181     NULL,
1182     &IsModified
1183     );
1184   ASSERT (IsModified == TRUE);
1185
1186 Done:
1187   mPageTablePoolLock = FALSE;
1188   return IsModified;
1189 }
1190
1191 /**
1192   This API provides a way to allocate memory for page table.
1193
1194   This API can be called more than once to allocate memory for page tables.
1195
1196   Allocates the number of 4KB pages and returns a pointer to the allocated
1197   buffer. The buffer returned is aligned on a 4KB boundary.
1198
1199   If Pages is 0, then NULL is returned.
1200   If there is not enough memory remaining to satisfy the request, then NULL is
1201   returned.
1202
1203   @param  Pages                 The number of 4 KB pages to allocate.
1204
1205   @return A pointer to the allocated buffer or NULL if allocation fails.
1206
1207 **/
1208 VOID *
1209 EFIAPI
1210 AllocatePageTableMemory (
1211   IN UINTN  Pages
1212   )
1213 {
1214   VOID  *Buffer;
1215
1216   if (Pages == 0) {
1217     return NULL;
1218   }
1219
1220   //
1221   // Renew the pool if necessary.
1222   //
1223   if ((mPageTablePool == NULL) ||
1224       (Pages > mPageTablePool->FreePages))
1225   {
1226     if (!InitializePageTablePool (Pages)) {
1227       return NULL;
1228     }
1229   }
1230
1231   Buffer = (UINT8 *)mPageTablePool + mPageTablePool->Offset;
1232
1233   mPageTablePool->Offset    += EFI_PAGES_TO_SIZE (Pages);
1234   mPageTablePool->FreePages -= Pages;
1235
1236   return Buffer;
1237 }
1238
1239 /**
1240   Special handler for #DB exception, which will restore the page attributes
1241   (not-present). It should work with #PF handler which will set pages to
1242   'present'.
1243
1244   @param ExceptionType  Exception type.
1245   @param SystemContext  Pointer to EFI_SYSTEM_CONTEXT.
1246
1247 **/
1248 VOID
1249 EFIAPI
1250 DebugExceptionHandler (
1251   IN EFI_EXCEPTION_TYPE  ExceptionType,
1252   IN EFI_SYSTEM_CONTEXT  SystemContext
1253   )
1254 {
1255   UINTN    CpuIndex;
1256   UINTN    PFEntry;
1257   BOOLEAN  IsWpEnabled;
1258
1259   MpInitLibWhoAmI (&CpuIndex);
1260
1261   //
1262   // Clear last PF entries
1263   //
1264   IsWpEnabled = IsReadOnlyPageWriteProtected ();
1265   if (IsWpEnabled) {
1266     DisableReadOnlyPageWriteProtect ();
1267   }
1268
1269   for (PFEntry = 0; PFEntry < mPFEntryCount[CpuIndex]; PFEntry++) {
1270     if (mLastPFEntryPointer[CpuIndex][PFEntry] != NULL) {
1271       *mLastPFEntryPointer[CpuIndex][PFEntry] &= ~(UINT64)IA32_PG_P;
1272     }
1273   }
1274
1275   if (IsWpEnabled) {
1276     EnableReadOnlyPageWriteProtect ();
1277   }
1278
1279   //
1280   // Reset page fault exception count for next page fault.
1281   //
1282   mPFEntryCount[CpuIndex] = 0;
1283
1284   //
1285   // Flush TLB
1286   //
1287   CpuFlushTlb ();
1288
1289   //
1290   // Clear TF in EFLAGS
1291   //
1292   if (mPagingContext.MachineType == IMAGE_FILE_MACHINE_I386) {
1293     SystemContext.SystemContextIa32->Eflags &= (UINT32) ~BIT8;
1294   } else {
1295     SystemContext.SystemContextX64->Rflags &= (UINT64) ~BIT8;
1296   }
1297 }
1298
1299 /**
1300   Special handler for #PF exception, which will set the pages which caused
1301   #PF to be 'present'. The attribute of those pages should be restored in
1302   the subsequent #DB handler.
1303
1304   @param ExceptionType  Exception type.
1305   @param SystemContext  Pointer to EFI_SYSTEM_CONTEXT.
1306
1307 **/
1308 VOID
1309 EFIAPI
1310 PageFaultExceptionHandler (
1311   IN EFI_EXCEPTION_TYPE  ExceptionType,
1312   IN EFI_SYSTEM_CONTEXT  SystemContext
1313   )
1314 {
1315   EFI_STATUS                     Status;
1316   UINT64                         PFAddress;
1317   PAGE_TABLE_LIB_PAGING_CONTEXT  PagingContext;
1318   PAGE_ATTRIBUTE                 PageAttribute;
1319   UINT64                         Attributes;
1320   UINT64                         *PageEntry;
1321   UINTN                          Index;
1322   UINTN                          CpuIndex;
1323   UINTN                          PageNumber;
1324   BOOLEAN                        NonStopMode;
1325
1326   PFAddress = AsmReadCr2 () & ~EFI_PAGE_MASK;
1327   if (PFAddress < BASE_4KB) {
1328     NonStopMode = NULL_DETECTION_NONSTOP_MODE ? TRUE : FALSE;
1329   } else {
1330     NonStopMode = HEAP_GUARD_NONSTOP_MODE ? TRUE : FALSE;
1331   }
1332
1333   if (NonStopMode) {
1334     MpInitLibWhoAmI (&CpuIndex);
1335     GetCurrentPagingContext (&PagingContext);
1336     //
1337     // Memory operation cross page boundary, like "rep mov" instruction, will
1338     // cause infinite loop between this and Debug Trap handler. We have to make
1339     // sure that current page and the page followed are both in PRESENT state.
1340     //
1341     PageNumber = 2;
1342     while (PageNumber > 0) {
1343       PageEntry = GetPageTableEntry (&PagingContext, PFAddress, &PageAttribute);
1344       ASSERT (PageEntry != NULL);
1345
1346       if (PageEntry != NULL) {
1347         Attributes = GetAttributesFromPageEntry (PageEntry);
1348         if ((Attributes & EFI_MEMORY_RP) != 0) {
1349           Attributes &= ~EFI_MEMORY_RP;
1350           Status      = AssignMemoryPageAttributes (
1351                           &PagingContext,
1352                           PFAddress,
1353                           EFI_PAGE_SIZE,
1354                           Attributes,
1355                           NULL
1356                           );
1357           if (!EFI_ERROR (Status)) {
1358             Index = mPFEntryCount[CpuIndex];
1359             //
1360             // Re-retrieve page entry because above calling might update page
1361             // table due to table split.
1362             //
1363             PageEntry                              = GetPageTableEntry (&PagingContext, PFAddress, &PageAttribute);
1364             mLastPFEntryPointer[CpuIndex][Index++] = PageEntry;
1365             mPFEntryCount[CpuIndex]                = Index;
1366           }
1367         }
1368       }
1369
1370       PFAddress += EFI_PAGE_SIZE;
1371       --PageNumber;
1372     }
1373   }
1374
1375   //
1376   // Initialize the serial port before dumping.
1377   //
1378   SerialPortInitialize ();
1379   //
1380   // Display ExceptionType, CPU information and Image information
1381   //
1382   DumpCpuContext (ExceptionType, SystemContext);
1383   if (NonStopMode) {
1384     //
1385     // Set TF in EFLAGS
1386     //
1387     if (mPagingContext.MachineType == IMAGE_FILE_MACHINE_I386) {
1388       SystemContext.SystemContextIa32->Eflags |= (UINT32)BIT8;
1389     } else {
1390       SystemContext.SystemContextX64->Rflags |= (UINT64)BIT8;
1391     }
1392   } else {
1393     CpuDeadLoop ();
1394   }
1395 }
1396
1397 /**
1398   Initialize the Page Table lib.
1399 **/
1400 VOID
1401 InitializePageTableLib (
1402   VOID
1403   )
1404 {
1405   PAGE_TABLE_LIB_PAGING_CONTEXT  CurrentPagingContext;
1406   UINT32                         *Attributes;
1407   UINTN                          *PageTableBase;
1408
1409   GetCurrentPagingContext (&CurrentPagingContext);
1410
1411   GetPagingDetails (&CurrentPagingContext.ContextData, &PageTableBase, &Attributes);
1412
1413   //
1414   // Reserve memory of page tables for future uses, if paging is enabled.
1415   //
1416   if ((*PageTableBase != 0) &&
1417       ((*Attributes & PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE) != 0))
1418   {
1419     DisableReadOnlyPageWriteProtect ();
1420     InitializePageTablePool (1);
1421     EnableReadOnlyPageWriteProtect ();
1422   }
1423
1424   if (HEAP_GUARD_NONSTOP_MODE || NULL_DETECTION_NONSTOP_MODE) {
1425     mPFEntryCount = (UINTN *)AllocateZeroPool (sizeof (UINTN) * mNumberOfProcessors);
1426     ASSERT (mPFEntryCount != NULL);
1427
1428     mLastPFEntryPointer = (UINT64 *(*)[MAX_PF_ENTRY_COUNT])
1429                           AllocateZeroPool (sizeof (mLastPFEntryPointer[0]) * mNumberOfProcessors);
1430     ASSERT (mLastPFEntryPointer != NULL);
1431   }
1432
1433   DEBUG ((DEBUG_INFO, "CurrentPagingContext:\n"));
1434   DEBUG ((DEBUG_INFO, "  MachineType   - 0x%x\n", CurrentPagingContext.MachineType));
1435   DEBUG ((DEBUG_INFO, "  PageTableBase - 0x%Lx\n", (UINT64)*PageTableBase));
1436   DEBUG ((DEBUG_INFO, "  Attributes    - 0x%x\n", *Attributes));
1437
1438   return;
1439 }