UefiCpuPkg/CpuDxe/CpuPageTable.c

   1 /** @file
   2   Page table management support.
   3
   4   Copyright (c) 2017 - 2019, Intel Corporation. All rights reserved.<BR>
   5   Copyright (c) 2017, AMD Incorporated. All rights reserved.<BR>
   6
   7   SPDX-License-Identifier: BSD-2-Clause-Patent
   8
   9 **/
  10
  11 #include <Base.h>
  12 #include <Uefi.h>
  13 #include <Library/PeCoffGetEntryPointLib.h>
  14 #include <Library/SerialPortLib.h>
  15 #include <Library/SynchronizationLib.h>
  16 #include <Library/PrintLib.h>
  17 #include <Protocol/SmmBase2.h>
  18 #include <Register/Intel/Cpuid.h>
  19 #include <Register/Intel/Msr.h>
  20
  21 #include "CpuDxe.h"
  22 #include "CpuPageTable.h"
  23
  24 ///
  25 /// Page Table Entry
  26 ///
  27 #define IA32_PG_P       BIT0
  28 #define IA32_PG_RW      BIT1
  29 #define IA32_PG_U       BIT2
  30 #define IA32_PG_WT      BIT3
  31 #define IA32_PG_CD      BIT4
  32 #define IA32_PG_A       BIT5
  33 #define IA32_PG_D       BIT6
  34 #define IA32_PG_PS      BIT7
  35 #define IA32_PG_PAT_2M  BIT12
  36 #define IA32_PG_PAT_4K  IA32_PG_PS
  37 #define IA32_PG_PMNT    BIT62
  38 #define IA32_PG_NX      BIT63
  39
  40 #define PAGE_ATTRIBUTE_BITS  (IA32_PG_D | IA32_PG_A | IA32_PG_U | IA32_PG_RW | IA32_PG_P)
  41 //
  42 // Bits 1, 2, 5, 6 are reserved in the IA32 PAE PDPTE
  43 // X64 PAE PDPTE does not have such restriction
  44 //
  45 #define IA32_PAE_PDPTE_ATTRIBUTE_BITS  (IA32_PG_P)
  46
  47 #define PAGE_PROGATE_BITS  (IA32_PG_NX | PAGE_ATTRIBUTE_BITS)
  48
  49 #define PAGING_4K_MASK  0xFFF
  50 #define PAGING_2M_MASK  0x1FFFFF
  51 #define PAGING_1G_MASK  0x3FFFFFFF
  52
  53 #define PAGING_PAE_INDEX_MASK  0x1FF
  54
  55 #define PAGING_4K_ADDRESS_MASK_64  0x000FFFFFFFFFF000ull
  56 #define PAGING_2M_ADDRESS_MASK_64  0x000FFFFFFFE00000ull
  57 #define PAGING_1G_ADDRESS_MASK_64  0x000FFFFFC0000000ull
  58
  59 #define MAX_PF_ENTRY_COUNT        10
  60 #define MAX_DEBUG_MESSAGE_LENGTH  0x100
  61 #define IA32_PF_EC_ID             BIT4
  62
  63 typedef enum {
  64   PageNone,
  65   Page4K,
  66   Page2M,
  67   Page1G,
  68 } PAGE_ATTRIBUTE;
  69
  70 typedef struct {
  71   PAGE_ATTRIBUTE    Attribute;
  72   UINT64            Length;
  73   UINT64            AddressMask;
  74 } PAGE_ATTRIBUTE_TABLE;
  75
  76 typedef enum {
  77   PageActionAssign,
  78   PageActionSet,
  79   PageActionClear,
  80 } PAGE_ACTION;
  81
  82 PAGE_ATTRIBUTE_TABLE  mPageAttributeTable[] = {
  83   { Page4K, SIZE_4KB, PAGING_4K_ADDRESS_MASK_64 },
  84   { Page2M, SIZE_2MB, PAGING_2M_ADDRESS_MASK_64 },
  85   { Page1G, SIZE_1GB, PAGING_1G_ADDRESS_MASK_64 },
  86 };
  87
  88 PAGE_TABLE_POOL                *mPageTablePool    = NULL;
  89 BOOLEAN                        mPageTablePoolLock = FALSE;
  90 PAGE_TABLE_LIB_PAGING_CONTEXT  mPagingContext;
  91 EFI_SMM_BASE2_PROTOCOL         *mSmmBase2 = NULL;
  92
  93 //
  94 // Record the page fault exception count for one instruction execution.
  95 //
  96 UINTN  *mPFEntryCount;
  97 UINT64                    *(*mLastPFEntryPointer)[MAX_PF_ENTRY_COUNT];
  98
  99 /**
 100  Check if current execution environment is in SMM mode or not, via
 101  EFI_SMM_BASE2_PROTOCOL.
 102
 103  This is necessary because of the fact that MdePkg\Library\SmmMemoryAllocationLib
 104  supports to free memory outside SMRAM. The library will call gBS->FreePool() or
 105  gBS->FreePages() and then SetMemorySpaceAttributes interface in turn to change
 106  memory paging attributes during free operation, if some memory related features
 107  are enabled (like Heap Guard).
 108
 109  This means that SetMemorySpaceAttributes() has chance to run in SMM mode. This
 110  will cause incorrect result because SMM mode always loads its own page tables,
 111  which are usually different from DXE. This function can be used to detect such
 112  situation and help to avoid further misoperations.
 113
 114   @retval TRUE    In SMM mode.
 115   @retval FALSE   Not in SMM mode.
 116 **/
 117 BOOLEAN
 118 IsInSmm (
 119   VOID
 120   )
 121 {
 122   BOOLEAN  InSmm;
 123
 124   InSmm = FALSE;
 125   if (mSmmBase2 == NULL) {
 126     gBS->LocateProtocol (&gEfiSmmBase2ProtocolGuid, NULL, (VOID **)&mSmmBase2);
 127   }
 128
 129   if (mSmmBase2 != NULL) {
 130     mSmmBase2->InSmm (mSmmBase2, &InSmm);
 131   }
 132
 133   //
 134   // mSmmBase2->InSmm() can only detect if the caller is running in SMRAM
 135   // or from SMM driver. It cannot tell if the caller is running in SMM mode.
 136   // Check page table base address to guarantee that because SMM mode willl
 137   // load its own page table.
 138   //
 139   return (InSmm &&
 140           mPagingContext.ContextData.X64.PageTableBase != (UINT64)AsmReadCr3 ());
 141 }
 142
 143 /**
 144   Return current paging context.
 145
 146   @param[in,out]  PagingContext     The paging context.
 147 **/
 148 VOID
 149 GetCurrentPagingContext (
 150   IN OUT PAGE_TABLE_LIB_PAGING_CONTEXT  *PagingContext
 151   )
 152 {
 153   UINT32                      RegEax;
 154   CPUID_EXTENDED_CPU_SIG_EDX  RegEdx;
 155   MSR_IA32_EFER_REGISTER      MsrEfer;
 156   IA32_CR4                    Cr4;
 157   IA32_CR0                    Cr0;
 158   UINT32                      *Attributes;
 159   UINTN                       *PageTableBase;
 160
 161   //
 162   // Don't retrieve current paging context from processor if in SMM mode.
 163   //
 164   if (!IsInSmm ()) {
 165     ZeroMem (&mPagingContext, sizeof (mPagingContext));
 166     if (sizeof (UINTN) == sizeof (UINT64)) {
 167       mPagingContext.MachineType = IMAGE_FILE_MACHINE_X64;
 168     } else {
 169       mPagingContext.MachineType = IMAGE_FILE_MACHINE_I386;
 170     }
 171
 172     GetPagingDetails (&mPagingContext.ContextData, &PageTableBase, &Attributes);
 173
 174     Cr0.UintN = AsmReadCr0 ();
 175     Cr4.UintN = AsmReadCr4 ();
 176
 177     if (Cr0.Bits.PG != 0) {
 178       *PageTableBase = (AsmReadCr3 () & PAGING_4K_ADDRESS_MASK_64);
 179     } else {
 180       *PageTableBase = 0;
 181     }
 182
 183     if (Cr0.Bits.WP  != 0) {
 184       *Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_WP_ENABLE;
 185     }
 186
 187     if (Cr4.Bits.PSE != 0) {
 188       *Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PSE;
 189     }
 190
 191     if (Cr4.Bits.PAE != 0) {
 192       *Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE;
 193     }
 194
 195     if (Cr4.Bits.LA57 != 0) {
 196       *Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_5_LEVEL;
 197     }
 198
 199     AsmCpuid (CPUID_EXTENDED_FUNCTION, &RegEax, NULL, NULL, NULL);
 200     if (RegEax >= CPUID_EXTENDED_CPU_SIG) {
 201       AsmCpuid (CPUID_EXTENDED_CPU_SIG, NULL, NULL, NULL, &RegEdx.Uint32);
 202
 203       if (RegEdx.Bits.NX != 0) {
 204         // XD supported
 205         MsrEfer.Uint64 = AsmReadMsr64 (MSR_CORE_IA32_EFER);
 206         if (MsrEfer.Bits.NXE != 0) {
 207           // XD activated
 208           *Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_XD_ACTIVATED;
 209         }
 210       }
 211
 212       if (RegEdx.Bits.Page1GB != 0) {
 213         *Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAGE_1G_SUPPORT;
 214       }
 215     }
 216   }
 217
 218   //
 219   // This can avoid getting SMM paging context if in SMM mode. We cannot assume
 220   // SMM mode shares the same paging context as DXE.
 221   //
 222   CopyMem (PagingContext, &mPagingContext, sizeof (mPagingContext));
 223 }
 224
 225 /**
 226   Return length according to page attributes.
 227
 228   @param[in]  PageAttributes   The page attribute of the page entry.
 229
 230   @return The length of page entry.
 231 **/
 232 UINTN
 233 PageAttributeToLength (
 234   IN PAGE_ATTRIBUTE  PageAttribute
 235   )
 236 {
 237   UINTN  Index;
 238
 239   for (Index = 0; Index < sizeof (mPageAttributeTable)/sizeof (mPageAttributeTable[0]); Index++) {
 240     if (PageAttribute == mPageAttributeTable[Index].Attribute) {
 241       return (UINTN)mPageAttributeTable[Index].Length;
 242     }
 243   }
 244
 245   return 0;
 246 }
 247
 248 /**
 249   Return address mask according to page attributes.
 250
 251   @param[in]  PageAttributes   The page attribute of the page entry.
 252
 253   @return The address mask of page entry.
 254 **/
 255 UINTN
 256 PageAttributeToMask (
 257   IN PAGE_ATTRIBUTE  PageAttribute
 258   )
 259 {
 260   UINTN  Index;
 261
 262   for (Index = 0; Index < sizeof (mPageAttributeTable)/sizeof (mPageAttributeTable[0]); Index++) {
 263     if (PageAttribute == mPageAttributeTable[Index].Attribute) {
 264       return (UINTN)mPageAttributeTable[Index].AddressMask;
 265     }
 266   }
 267
 268   return 0;
 269 }
 270
 271 /**
 272   Return page table entry to match the address.
 273
 274   @param[in]  PagingContext     The paging context.
 275   @param[in]  Address           The address to be checked.
 276   @param[out] PageAttributes    The page attribute of the page entry.
 277
 278   @return The page entry.
 279 **/
 280 VOID *
 281 GetPageTableEntry (
 282   IN  PAGE_TABLE_LIB_PAGING_CONTEXT  *PagingContext,
 283   IN  PHYSICAL_ADDRESS               Address,
 284   OUT PAGE_ATTRIBUTE                 *PageAttribute
 285   )
 286 {
 287   UINTN   Index1;
 288   UINTN   Index2;
 289   UINTN   Index3;
 290   UINTN   Index4;
 291   UINTN   Index5;
 292   UINT64  *L1PageTable;
 293   UINT64  *L2PageTable;
 294   UINT64  *L3PageTable;
 295   UINT64  *L4PageTable;
 296   UINT64  *L5PageTable;
 297   UINT64  AddressEncMask;
 298
 299   ASSERT (PagingContext != NULL);
 300
 301   Index5 = ((UINTN)RShiftU64 (Address, 48)) & PAGING_PAE_INDEX_MASK;
 302   Index4 = ((UINTN)RShiftU64 (Address, 39)) & PAGING_PAE_INDEX_MASK;
 303   Index3 = ((UINTN)Address >> 30) & PAGING_PAE_INDEX_MASK;
 304   Index2 = ((UINTN)Address >> 21) & PAGING_PAE_INDEX_MASK;
 305   Index1 = ((UINTN)Address >> 12) & PAGING_PAE_INDEX_MASK;
 306
 307   // Make sure AddressEncMask is contained to smallest supported address field.
 308   //
 309   AddressEncMask = PcdGet64 (PcdPteMemoryEncryptionAddressOrMask) & PAGING_1G_ADDRESS_MASK_64;
 310   if (AddressEncMask == 0) {
 311     AddressEncMask = PcdGet64 (PcdTdxSharedBitMask) & PAGING_1G_ADDRESS_MASK_64;
 312   }
 313
 314   if (PagingContext->MachineType == IMAGE_FILE_MACHINE_X64) {
 315     if ((PagingContext->ContextData.X64.Attributes & PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_5_LEVEL) != 0) {
 316       L5PageTable = (UINT64 *)(UINTN)PagingContext->ContextData.X64.PageTableBase;
 317       if (L5PageTable[Index5] == 0) {
 318         *PageAttribute = PageNone;
 319         return NULL;
 320       }
 321
 322       L4PageTable = (UINT64 *)(UINTN)(L5PageTable[Index5] & ~AddressEncMask & PAGING_4K_ADDRESS_MASK_64);
 323     } else {
 324       L4PageTable = (UINT64 *)(UINTN)PagingContext->ContextData.X64.PageTableBase;
 325     }
 326
 327     if (L4PageTable[Index4] == 0) {
 328       *PageAttribute = PageNone;
 329       return NULL;
 330     }
 331
 332     L3PageTable = (UINT64 *)(UINTN)(L4PageTable[Index4] & ~AddressEncMask & PAGING_4K_ADDRESS_MASK_64);
 333   } else {
 334     ASSERT ((PagingContext->ContextData.Ia32.Attributes & PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE) != 0);
 335     L3PageTable = (UINT64 *)(UINTN)PagingContext->ContextData.Ia32.PageTableBase;
 336   }
 337
 338   if (L3PageTable[Index3] == 0) {
 339     *PageAttribute = PageNone;
 340     return NULL;
 341   }
 342
 343   if ((L3PageTable[Index3] & IA32_PG_PS) != 0) {
 344     // 1G
 345     *PageAttribute = Page1G;
 346     return &L3PageTable[Index3];
 347   }
 348
 349   L2PageTable = (UINT64 *)(UINTN)(L3PageTable[Index3] & ~AddressEncMask & PAGING_4K_ADDRESS_MASK_64);
 350   if (L2PageTable[Index2] == 0) {
 351     *PageAttribute = PageNone;
 352     return NULL;
 353   }
 354
 355   if ((L2PageTable[Index2] & IA32_PG_PS) != 0) {
 356     // 2M
 357     *PageAttribute = Page2M;
 358     return &L2PageTable[Index2];
 359   }
 360
 361   // 4k
 362   L1PageTable = (UINT64 *)(UINTN)(L2PageTable[Index2] & ~AddressEncMask & PAGING_4K_ADDRESS_MASK_64);
 363   if ((L1PageTable[Index1] == 0) && (Address != 0)) {
 364     *PageAttribute = PageNone;
 365     return NULL;
 366   }
 367
 368   *PageAttribute = Page4K;
 369   return &L1PageTable[Index1];
 370 }
 371
 372 /**
 373   Return memory attributes of page entry.
 374
 375   @param[in]  PageEntry        The page entry.
 376
 377   @return Memory attributes of page entry.
 378 **/
 379 UINT64
 380 GetAttributesFromPageEntry (
 381   IN  UINT64  *PageEntry
 382   )
 383 {
 384   UINT64  Attributes;
 385
 386   Attributes = 0;
 387   if ((*PageEntry & IA32_PG_P) == 0) {
 388     Attributes |= EFI_MEMORY_RP;
 389   }
 390
 391   if ((*PageEntry & IA32_PG_RW) == 0) {
 392     Attributes |= EFI_MEMORY_RO;
 393   }
 394
 395   if ((*PageEntry & IA32_PG_NX) != 0) {
 396     Attributes |= EFI_MEMORY_XP;
 397   }
 398
 399   return Attributes;
 400 }
 401
 402 /**
 403   Modify memory attributes of page entry.
 404
 405   @param[in]  PagingContext    The paging context.
 406   @param[in]  PageEntry        The page entry.
 407   @param[in]  Attributes       The bit mask of attributes to modify for the memory region.
 408   @param[in]  PageAction       The page action.
 409   @param[out] IsModified       TRUE means page table modified. FALSE means page table not modified.
 410 **/
 411 VOID
 412 ConvertPageEntryAttribute (
 413   IN  PAGE_TABLE_LIB_PAGING_CONTEXT  *PagingContext,
 414   IN  UINT64                         *PageEntry,
 415   IN  UINT64                         Attributes,
 416   IN  PAGE_ACTION                    PageAction,
 417   OUT BOOLEAN                        *IsModified
 418   )
 419 {
 420   UINT64  CurrentPageEntry;
 421   UINT64  NewPageEntry;
 422   UINT32  *PageAttributes;
 423
 424   CurrentPageEntry = *PageEntry;
 425   NewPageEntry     = CurrentPageEntry;
 426   if ((Attributes & EFI_MEMORY_RP) != 0) {
 427     switch (PageAction) {
 428       case PageActionAssign:
 429       case PageActionSet:
 430         NewPageEntry &= ~(UINT64)IA32_PG_P;
 431         break;
 432       case PageActionClear:
 433         NewPageEntry |= IA32_PG_P;
 434         break;
 435     }
 436   } else {
 437     switch (PageAction) {
 438       case PageActionAssign:
 439         NewPageEntry |= IA32_PG_P;
 440         break;
 441       case PageActionSet:
 442       case PageActionClear:
 443         break;
 444     }
 445   }
 446
 447   if ((Attributes & EFI_MEMORY_RO) != 0) {
 448     switch (PageAction) {
 449       case PageActionAssign:
 450       case PageActionSet:
 451         NewPageEntry &= ~(UINT64)IA32_PG_RW;
 452         break;
 453       case PageActionClear:
 454         NewPageEntry |= IA32_PG_RW;
 455         break;
 456     }
 457   } else {
 458     switch (PageAction) {
 459       case PageActionAssign:
 460         NewPageEntry |= IA32_PG_RW;
 461         break;
 462       case PageActionSet:
 463       case PageActionClear:
 464         break;
 465     }
 466   }
 467
 468   GetPagingDetails (&PagingContext->ContextData, NULL, &PageAttributes);
 469
 470   if ((*PageAttributes & PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_XD_ACTIVATED) != 0) {
 471     if ((Attributes & EFI_MEMORY_XP) != 0) {
 472       switch (PageAction) {
 473         case PageActionAssign:
 474         case PageActionSet:
 475           NewPageEntry |= IA32_PG_NX;
 476           break;
 477         case PageActionClear:
 478           NewPageEntry &= ~IA32_PG_NX;
 479           break;
 480       }
 481     } else {
 482       switch (PageAction) {
 483         case PageActionAssign:
 484           NewPageEntry &= ~IA32_PG_NX;
 485           break;
 486         case PageActionSet:
 487         case PageActionClear:
 488           break;
 489       }
 490     }
 491   }
 492
 493   *PageEntry = NewPageEntry;
 494   if (CurrentPageEntry != NewPageEntry) {
 495     *IsModified = TRUE;
 496     DEBUG ((DEBUG_VERBOSE, "ConvertPageEntryAttribute 0x%lx", CurrentPageEntry));
 497     DEBUG ((DEBUG_VERBOSE, "->0x%lx\n", NewPageEntry));
 498   } else {
 499     *IsModified = FALSE;
 500   }
 501 }
 502
 503 /**
 504   This function returns if there is need to split page entry.
 505
 506   @param[in]  BaseAddress      The base address to be checked.
 507   @param[in]  Length           The length to be checked.
 508   @param[in]  PageEntry        The page entry to be checked.
 509   @param[in]  PageAttribute    The page attribute of the page entry.
 510
 511   @retval SplitAttributes on if there is need to split page entry.
 512 **/
 513 PAGE_ATTRIBUTE
 514 NeedSplitPage (
 515   IN  PHYSICAL_ADDRESS  BaseAddress,
 516   IN  UINT64            Length,
 517   IN  UINT64            *PageEntry,
 518   IN  PAGE_ATTRIBUTE    PageAttribute
 519   )
 520 {
 521   UINT64  PageEntryLength;
 522
 523   PageEntryLength = PageAttributeToLength (PageAttribute);
 524
 525   if (((BaseAddress & (PageEntryLength - 1)) == 0) && (Length >= PageEntryLength)) {
 526     return PageNone;
 527   }
 528
 529   if (((BaseAddress & PAGING_2M_MASK) != 0) || (Length < SIZE_2MB)) {
 530     return Page4K;
 531   }
 532
 533   return Page2M;
 534 }
 535
 536 /**
 537   This function splits one page entry to small page entries.
 538
 539   @param[in]  PageEntry         The page entry to be splitted.
 540   @param[in]  PageAttribute     The page attribute of the page entry.
 541   @param[in]  SplitAttribute    How to split the page entry.
 542   @param[in]  AllocatePagesFunc If page split is needed, this function is used to allocate more pages.
 543
 544   @retval RETURN_SUCCESS            The page entry is splitted.
 545   @retval RETURN_UNSUPPORTED        The page entry does not support to be splitted.
 546   @retval RETURN_OUT_OF_RESOURCES   No resource to split page entry.
 547 **/
 548 RETURN_STATUS
 549 SplitPage (
 550   IN  UINT64                         *PageEntry,
 551   IN  PAGE_ATTRIBUTE                 PageAttribute,
 552   IN  PAGE_ATTRIBUTE                 SplitAttribute,
 553   IN  PAGE_TABLE_LIB_ALLOCATE_PAGES  AllocatePagesFunc
 554   )
 555 {
 556   UINT64  BaseAddress;
 557   UINT64  *NewPageEntry;
 558   UINTN   Index;
 559   UINT64  AddressEncMask;
 560
 561   ASSERT (PageAttribute == Page2M || PageAttribute == Page1G);
 562
 563   ASSERT (AllocatePagesFunc != NULL);
 564
 565   // Make sure AddressEncMask is contained to smallest supported address field.
 566   //
 567   AddressEncMask = PcdGet64 (PcdPteMemoryEncryptionAddressOrMask) & PAGING_1G_ADDRESS_MASK_64;
 568
 569   if (PageAttribute == Page2M) {
 570     //
 571     // Split 2M to 4K
 572     //
 573     ASSERT (SplitAttribute == Page4K);
 574     if (SplitAttribute == Page4K) {
 575       NewPageEntry = AllocatePagesFunc (1);
 576       DEBUG ((DEBUG_VERBOSE, "Split - 0x%x\n", NewPageEntry));
 577       if (NewPageEntry == NULL) {
 578         return RETURN_OUT_OF_RESOURCES;
 579       }
 580
 581       BaseAddress = *PageEntry & ~AddressEncMask & PAGING_2M_ADDRESS_MASK_64;
 582       for (Index = 0; Index < SIZE_4KB / sizeof (UINT64); Index++) {
 583         NewPageEntry[Index] = (BaseAddress + SIZE_4KB * Index) | AddressEncMask | ((*PageEntry) & PAGE_PROGATE_BITS);
 584       }
 585
 586       (*PageEntry) = (UINT64)(UINTN)NewPageEntry | AddressEncMask | ((*PageEntry) & PAGE_ATTRIBUTE_BITS);
 587       return RETURN_SUCCESS;
 588     } else {
 589       return RETURN_UNSUPPORTED;
 590     }
 591   } else if (PageAttribute == Page1G) {
 592     //
 593     // Split 1G to 2M
 594     // No need support 1G->4K directly, we should use 1G->2M, then 2M->4K to get more compact page table.
 595     //
 596     ASSERT (SplitAttribute == Page2M || SplitAttribute == Page4K);
 597     if (((SplitAttribute == Page2M) || (SplitAttribute == Page4K))) {
 598       NewPageEntry = AllocatePagesFunc (1);
 599       DEBUG ((DEBUG_VERBOSE, "Split - 0x%x\n", NewPageEntry));
 600       if (NewPageEntry == NULL) {
 601         return RETURN_OUT_OF_RESOURCES;
 602       }
 603
 604       BaseAddress = *PageEntry & ~AddressEncMask  & PAGING_1G_ADDRESS_MASK_64;
 605       for (Index = 0; Index < SIZE_4KB / sizeof (UINT64); Index++) {
 606         NewPageEntry[Index] = (BaseAddress + SIZE_2MB * Index) | AddressEncMask | IA32_PG_PS | ((*PageEntry) & PAGE_PROGATE_BITS);
 607       }
 608
 609       (*PageEntry) = (UINT64)(UINTN)NewPageEntry | AddressEncMask | ((*PageEntry) & PAGE_ATTRIBUTE_BITS);
 610       return RETURN_SUCCESS;
 611     } else {
 612       return RETURN_UNSUPPORTED;
 613     }
 614   } else {
 615     return RETURN_UNSUPPORTED;
 616   }
 617 }
 618
 619 /**
 620  Check the WP status in CR0 register. This bit is used to lock or unlock write
 621  access to pages marked as read-only.
 622
 623   @retval TRUE    Write protection is enabled.
 624   @retval FALSE   Write protection is disabled.
 625 **/
 626 BOOLEAN
 627 IsReadOnlyPageWriteProtected (
 628   VOID
 629   )
 630 {
 631   IA32_CR0  Cr0;
 632
 633   //
 634   // To avoid unforseen consequences, don't touch paging settings in SMM mode
 635   // in this driver.
 636   //
 637   if (!IsInSmm ()) {
 638     Cr0.UintN = AsmReadCr0 ();
 639     return (BOOLEAN)(Cr0.Bits.WP != 0);
 640   }
 641
 642   return FALSE;
 643 }
 644
 645 /**
 646  Disable Write Protect on pages marked as read-only.
 647 **/
 648 VOID
 649 DisableReadOnlyPageWriteProtect (
 650   VOID
 651   )
 652 {
 653   IA32_CR0  Cr0;
 654
 655   //
 656   // To avoid unforseen consequences, don't touch paging settings in SMM mode
 657   // in this driver.
 658   //
 659   if (!IsInSmm ()) {
 660     Cr0.UintN   = AsmReadCr0 ();
 661     Cr0.Bits.WP = 0;
 662     AsmWriteCr0 (Cr0.UintN);
 663   }
 664 }
 665
 666 /**
 667  Enable Write Protect on pages marked as read-only.
 668 **/
 669 VOID
 670 EnableReadOnlyPageWriteProtect (
 671   VOID
 672   )
 673 {
 674   IA32_CR0  Cr0;
 675
 676   //
 677   // To avoid unforseen consequences, don't touch paging settings in SMM mode
 678   // in this driver.
 679   //
 680   if (!IsInSmm ()) {
 681     Cr0.UintN   = AsmReadCr0 ();
 682     Cr0.Bits.WP = 1;
 683     AsmWriteCr0 (Cr0.UintN);
 684   }
 685 }
 686
 687 /**
 688   This function modifies the page attributes for the memory region specified by BaseAddress and
 689   Length from their current attributes to the attributes specified by Attributes.
 690
 691   Caller should make sure BaseAddress and Length is at page boundary.
 692
 693   @param[in]  PagingContext     The paging context. NULL means get page table from current CPU context.
 694   @param[in]  BaseAddress       The physical address that is the start address of a memory region.
 695   @param[in]  Length            The size in bytes of the memory region.
 696   @param[in]  Attributes        The bit mask of attributes to modify for the memory region.
 697   @param[in]  PageAction        The page action.
 698   @param[in]  AllocatePagesFunc If page split is needed, this function is used to allocate more pages.
 699                                 NULL mean page split is unsupported.
 700   @param[out] IsSplitted        TRUE means page table splitted. FALSE means page table not splitted.
 701   @param[out] IsModified        TRUE means page table modified. FALSE means page table not modified.
 702
 703   @retval RETURN_SUCCESS           The attributes were modified for the memory region.
 704   @retval RETURN_ACCESS_DENIED     The attributes for the memory resource range specified by
 705                                    BaseAddress and Length cannot be modified.
 706   @retval RETURN_INVALID_PARAMETER Length is zero.
 707                                    Attributes specified an illegal combination of attributes that
 708                                    cannot be set together.
 709   @retval RETURN_OUT_OF_RESOURCES  There are not enough system resources to modify the attributes of
 710                                    the memory resource range.
 711   @retval RETURN_UNSUPPORTED       The processor does not support one or more bytes of the memory
 712                                    resource range specified by BaseAddress and Length.
 713                                    The bit mask of attributes is not support for the memory resource
 714                                    range specified by BaseAddress and Length.
 715 **/
 716 RETURN_STATUS
 717 ConvertMemoryPageAttributes (
 718   IN  PAGE_TABLE_LIB_PAGING_CONTEXT  *PagingContext OPTIONAL,
 719   IN  PHYSICAL_ADDRESS               BaseAddress,
 720   IN  UINT64                         Length,
 721   IN  UINT64                         Attributes,
 722   IN  PAGE_ACTION                    PageAction,
 723   IN  PAGE_TABLE_LIB_ALLOCATE_PAGES  AllocatePagesFunc OPTIONAL,
 724   OUT BOOLEAN                        *IsSplitted   OPTIONAL,
 725   OUT BOOLEAN                        *IsModified   OPTIONAL
 726   )
 727 {
 728   PAGE_TABLE_LIB_PAGING_CONTEXT  CurrentPagingContext;
 729   UINT64                         *PageEntry;
 730   PAGE_ATTRIBUTE                 PageAttribute;
 731   UINTN                          PageEntryLength;
 732   PAGE_ATTRIBUTE                 SplitAttribute;
 733   RETURN_STATUS                  Status;
 734   BOOLEAN                        IsEntryModified;
 735   BOOLEAN                        IsWpEnabled;
 736
 737   if ((BaseAddress & (SIZE_4KB - 1)) != 0) {
 738     DEBUG ((DEBUG_ERROR, "BaseAddress(0x%lx) is not aligned!\n", BaseAddress));
 739     return EFI_UNSUPPORTED;
 740   }
 741
 742   if ((Length & (SIZE_4KB - 1)) != 0) {
 743     DEBUG ((DEBUG_ERROR, "Length(0x%lx) is not aligned!\n", Length));
 744     return EFI_UNSUPPORTED;
 745   }
 746
 747   if (Length == 0) {
 748     DEBUG ((DEBUG_ERROR, "Length is 0!\n"));
 749     return RETURN_INVALID_PARAMETER;
 750   }
 751
 752   if ((Attributes & ~EFI_MEMORY_ATTRIBUTE_MASK) != 0) {
 753     DEBUG ((DEBUG_ERROR, "Attributes(0x%lx) has unsupported bit\n", Attributes));
 754     return EFI_UNSUPPORTED;
 755   }
 756
 757   if (PagingContext == NULL) {
 758     GetCurrentPagingContext (&CurrentPagingContext);
 759   } else {
 760     CopyMem (&CurrentPagingContext, PagingContext, sizeof (CurrentPagingContext));
 761   }
 762
 763   switch (CurrentPagingContext.MachineType) {
 764     case IMAGE_FILE_MACHINE_I386:
 765       if (CurrentPagingContext.ContextData.Ia32.PageTableBase == 0) {
 766         if (Attributes == 0) {
 767           return EFI_SUCCESS;
 768         } else {
 769           DEBUG ((DEBUG_ERROR, "PageTable is 0!\n"));
 770           return EFI_UNSUPPORTED;
 771         }
 772       }
 773
 774       if ((CurrentPagingContext.ContextData.Ia32.Attributes & PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE) == 0) {
 775         DEBUG ((DEBUG_ERROR, "Non-PAE Paging!\n"));
 776         return EFI_UNSUPPORTED;
 777       }
 778
 779       if ((BaseAddress + Length) > BASE_4GB) {
 780         DEBUG ((DEBUG_ERROR, "Beyond 4GB memory in 32-bit mode!\n"));
 781         return EFI_UNSUPPORTED;
 782       }
 783
 784       break;
 785     case IMAGE_FILE_MACHINE_X64:
 786       ASSERT (CurrentPagingContext.ContextData.X64.PageTableBase != 0);
 787       break;
 788     default:
 789       ASSERT (FALSE);
 790       return EFI_UNSUPPORTED;
 791       break;
 792   }
 793
 794   //  DEBUG ((DEBUG_ERROR, "ConvertMemoryPageAttributes(%x) - %016lx, %016lx, %02lx\n", IsSet, BaseAddress, Length, Attributes));
 795
 796   if (IsSplitted != NULL) {
 797     *IsSplitted = FALSE;
 798   }
 799
 800   if (IsModified != NULL) {
 801     *IsModified = FALSE;
 802   }
 803
 804   if (AllocatePagesFunc == NULL) {
 805     AllocatePagesFunc = AllocatePageTableMemory;
 806   }
 807
 808   //
 809   // Make sure that the page table is changeable.
 810   //
 811   IsWpEnabled = IsReadOnlyPageWriteProtected ();
 812   if (IsWpEnabled) {
 813     DisableReadOnlyPageWriteProtect ();
 814   }
 815
 816   //
 817   // Below logic is to check 2M/4K page to make sure we do not waste memory.
 818   //
 819   Status = EFI_SUCCESS;
 820   while (Length != 0) {
 821     PageEntry = GetPageTableEntry (&CurrentPagingContext, BaseAddress, &PageAttribute);
 822     if (PageEntry == NULL) {
 823       Status = RETURN_UNSUPPORTED;
 824       goto Done;
 825     }
 826
 827     PageEntryLength = PageAttributeToLength (PageAttribute);
 828     SplitAttribute  = NeedSplitPage (BaseAddress, Length, PageEntry, PageAttribute);
 829     if (SplitAttribute == PageNone) {
 830       ConvertPageEntryAttribute (&CurrentPagingContext, PageEntry, Attributes, PageAction, &IsEntryModified);
 831       if (IsEntryModified) {
 832         if (IsModified != NULL) {
 833           *IsModified = TRUE;
 834         }
 835       }
 836
 837       //
 838       // Convert success, move to next
 839       //
 840       BaseAddress += PageEntryLength;
 841       Length      -= PageEntryLength;
 842     } else {
 843       if (AllocatePagesFunc == NULL) {
 844         Status = RETURN_UNSUPPORTED;
 845         goto Done;
 846       }
 847
 848       Status = SplitPage (PageEntry, PageAttribute, SplitAttribute, AllocatePagesFunc);
 849       if (RETURN_ERROR (Status)) {
 850         Status = RETURN_UNSUPPORTED;
 851         goto Done;
 852       }
 853
 854       if (IsSplitted != NULL) {
 855         *IsSplitted = TRUE;
 856       }
 857
 858       if (IsModified != NULL) {
 859         *IsModified = TRUE;
 860       }
 861
 862       //
 863       // Just split current page
 864       // Convert success in next around
 865       //
 866     }
 867   }
 868
 869 Done:
 870   //
 871   // Restore page table write protection, if any.
 872   //
 873   if (IsWpEnabled) {
 874     EnableReadOnlyPageWriteProtect ();
 875   }
 876
 877   return Status;
 878 }
 879
 880 /**
 881   This function assigns the page attributes for the memory region specified by BaseAddress and
 882   Length from their current attributes to the attributes specified by Attributes.
 883
 884   Caller should make sure BaseAddress and Length is at page boundary.
 885
 886   Caller need guarantee the TPL <= TPL_NOTIFY, if there is split page request.
 887
 888   @param[in]  PagingContext     The paging context. NULL means get page table from current CPU context.
 889   @param[in]  BaseAddress       The physical address that is the start address of a memory region.
 890   @param[in]  Length            The size in bytes of the memory region.
 891   @param[in]  Attributes        The bit mask of attributes to set for the memory region.
 892   @param[in]  AllocatePagesFunc If page split is needed, this function is used to allocate more pages.
 893                                 NULL mean page split is unsupported.
 894
 895   @retval RETURN_SUCCESS           The attributes were cleared for the memory region.
 896   @retval RETURN_ACCESS_DENIED     The attributes for the memory resource range specified by
 897                                    BaseAddress and Length cannot be modified.
 898   @retval RETURN_INVALID_PARAMETER Length is zero.
 899                                    Attributes specified an illegal combination of attributes that
 900                                    cannot be set together.
 901   @retval RETURN_OUT_OF_RESOURCES  There are not enough system resources to modify the attributes of
 902                                    the memory resource range.
 903   @retval RETURN_UNSUPPORTED       The processor does not support one or more bytes of the memory
 904                                    resource range specified by BaseAddress and Length.
 905                                    The bit mask of attributes is not support for the memory resource
 906                                    range specified by BaseAddress and Length.
 907 **/
 908 RETURN_STATUS
 909 EFIAPI
 910 AssignMemoryPageAttributes (
 911   IN  PAGE_TABLE_LIB_PAGING_CONTEXT  *PagingContext OPTIONAL,
 912   IN  PHYSICAL_ADDRESS               BaseAddress,
 913   IN  UINT64                         Length,
 914   IN  UINT64                         Attributes,
 915   IN  PAGE_TABLE_LIB_ALLOCATE_PAGES  AllocatePagesFunc OPTIONAL
 916   )
 917 {
 918   RETURN_STATUS  Status;
 919   BOOLEAN        IsModified;
 920   BOOLEAN        IsSplitted;
 921
 922   //  DEBUG((DEBUG_INFO, "AssignMemoryPageAttributes: 0x%lx - 0x%lx (0x%lx)\n", BaseAddress, Length, Attributes));
 923   Status = ConvertMemoryPageAttributes (PagingContext, BaseAddress, Length, Attributes, PageActionAssign, AllocatePagesFunc, &IsSplitted, &IsModified);
 924   if (!EFI_ERROR (Status)) {
 925     if ((PagingContext == NULL) && IsModified) {
 926       //
 927       // Flush TLB as last step.
 928       //
 929       // Note: Since APs will always init CR3 register in HLT loop mode or do
 930       // TLB flush in MWAIT loop mode, there's no need to flush TLB for them
 931       // here.
 932       //
 933       CpuFlushTlb ();
 934     }
 935   }
 936
 937   return Status;
 938 }
 939
 940 /**
 941  Check if Execute Disable feature is enabled or not.
 942 **/
 943 BOOLEAN
 944 IsExecuteDisableEnabled (
 945   VOID
 946   )
 947 {
 948   MSR_CORE_IA32_EFER_REGISTER  MsrEfer;
 949
 950   MsrEfer.Uint64 = AsmReadMsr64 (MSR_IA32_EFER);
 951   return (MsrEfer.Bits.NXE == 1);
 952 }
 953
 954 /**
 955   Update GCD memory space attributes according to current page table setup.
 956 **/
 957 VOID
 958 RefreshGcdMemoryAttributesFromPaging (
 959   VOID
 960   )
 961 {
 962   EFI_STATUS                       Status;
 963   UINTN                            NumberOfDescriptors;
 964   EFI_GCD_MEMORY_SPACE_DESCRIPTOR  *MemorySpaceMap;
 965   PAGE_TABLE_LIB_PAGING_CONTEXT    PagingContext;
 966   PAGE_ATTRIBUTE                   PageAttribute;
 967   UINT64                           *PageEntry;
 968   UINT64                           PageLength;
 969   UINT64                           MemorySpaceLength;
 970   UINT64                           Length;
 971   UINT64                           BaseAddress;
 972   UINT64                           PageStartAddress;
 973   UINT64                           Attributes;
 974   UINT64                           Capabilities;
 975   UINT64                           NewAttributes;
 976   UINTN                            Index;
 977
 978   //
 979   // Assuming that memory space map returned is sorted already; otherwise sort
 980   // them in the order of lowest address to highest address.
 981   //
 982   Status = gDS->GetMemorySpaceMap (&NumberOfDescriptors, &MemorySpaceMap);
 983   ASSERT_EFI_ERROR (Status);
 984
 985   GetCurrentPagingContext (&PagingContext);
 986
 987   Attributes    = 0;
 988   NewAttributes = 0;
 989   BaseAddress   = 0;
 990   PageLength    = 0;
 991
 992   if (IsExecuteDisableEnabled ()) {
 993     Capabilities = EFI_MEMORY_RO | EFI_MEMORY_RP | EFI_MEMORY_XP;
 994   } else {
 995     Capabilities = EFI_MEMORY_RO | EFI_MEMORY_RP;
 996   }
 997
 998   for (Index = 0; Index < NumberOfDescriptors; Index++) {
 999     if (MemorySpaceMap[Index].GcdMemoryType == EfiGcdMemoryTypeNonExistent) {
1000       continue;
1001     }
1002
1003     //
1004     // Sync the actual paging related capabilities back to GCD service first.
1005     // As a side effect (good one), this can also help to avoid unnecessary
1006     // memory map entries due to the different capabilities of the same type
1007     // memory, such as multiple RT_CODE and RT_DATA entries in memory map,
1008     // which could cause boot failure of some old Linux distro (before v4.3).
1009     //
1010     Status = gDS->SetMemorySpaceCapabilities (
1011                     MemorySpaceMap[Index].BaseAddress,
1012                     MemorySpaceMap[Index].Length,
1013                     MemorySpaceMap[Index].Capabilities | Capabilities
1014                     );
1015     if (EFI_ERROR (Status)) {
1016       //
1017       // If we cannot update the capabilities, we cannot update its
1018       // attributes either. So just simply skip current block of memory.
1019       //
1020       DEBUG ((
1021         DEBUG_WARN,
1022         "Failed to update capability: [%lu] %016lx - %016lx (%016lx -> %016lx)\r\n",
1023         (UINT64)Index,
1024         MemorySpaceMap[Index].BaseAddress,
1025         MemorySpaceMap[Index].BaseAddress + MemorySpaceMap[Index].Length - 1,
1026         MemorySpaceMap[Index].Capabilities,
1027         MemorySpaceMap[Index].Capabilities | Capabilities
1028         ));
1029       continue;
1030     }
1031
1032     if (MemorySpaceMap[Index].BaseAddress >= (BaseAddress + PageLength)) {
1033       //
1034       // Current memory space starts at a new page. Resetting PageLength will
1035       // trigger a retrieval of page attributes at new address.
1036       //
1037       PageLength = 0;
1038     } else {
1039       //
1040       // In case current memory space is not adjacent to last one
1041       //
1042       PageLength -= (MemorySpaceMap[Index].BaseAddress - BaseAddress);
1043     }
1044
1045     //
1046     // Sync actual page attributes to GCD
1047     //
1048     BaseAddress       = MemorySpaceMap[Index].BaseAddress;
1049     MemorySpaceLength = MemorySpaceMap[Index].Length;
1050     while (MemorySpaceLength > 0) {
1051       if (PageLength == 0) {
1052         PageEntry = GetPageTableEntry (&PagingContext, BaseAddress, &PageAttribute);
1053         if (PageEntry == NULL) {
1054           break;
1055         }
1056
1057         //
1058         // Note current memory space might start in the middle of a page
1059         //
1060         PageStartAddress = (*PageEntry) & (UINT64)PageAttributeToMask (PageAttribute);
1061         PageLength       = PageAttributeToLength (PageAttribute) - (BaseAddress - PageStartAddress);
1062         Attributes       = GetAttributesFromPageEntry (PageEntry);
1063       }
1064
1065       Length = MIN (PageLength, MemorySpaceLength);
1066       if (Attributes != (MemorySpaceMap[Index].Attributes &
1067                          EFI_MEMORY_ATTRIBUTE_MASK))
1068       {
1069         NewAttributes = (MemorySpaceMap[Index].Attributes &
1070                          ~EFI_MEMORY_ATTRIBUTE_MASK) | Attributes;
1071         Status = gDS->SetMemorySpaceAttributes (
1072                         BaseAddress,
1073                         Length,
1074                         NewAttributes
1075                         );
1076         ASSERT_EFI_ERROR (Status);
1077         DEBUG ((
1078           DEBUG_VERBOSE,
1079           "Updated memory space attribute: [%lu] %016lx - %016lx (%016lx -> %016lx)\r\n",
1080           (UINT64)Index,
1081           BaseAddress,
1082           BaseAddress + Length - 1,
1083           MemorySpaceMap[Index].Attributes,
1084           NewAttributes
1085           ));
1086       }
1087
1088       PageLength        -= Length;
1089       MemorySpaceLength -= Length;
1090       BaseAddress       += Length;
1091     }
1092   }
1093
1094   FreePool (MemorySpaceMap);
1095 }
1096
1097 /**
1098   Initialize a buffer pool for page table use only.
1099
1100   To reduce the potential split operation on page table, the pages reserved for
1101   page table should be allocated in the times of PAGE_TABLE_POOL_UNIT_PAGES and
1102   at the boundary of PAGE_TABLE_POOL_ALIGNMENT. So the page pool is always
1103   initialized with number of pages greater than or equal to the given PoolPages.
1104
1105   Once the pages in the pool are used up, this method should be called again to
1106   reserve at least another PAGE_TABLE_POOL_UNIT_PAGES. Usually this won't happen
1107   often in practice.
1108
1109   @param[in] PoolPages      The least page number of the pool to be created.
1110
1111   @retval TRUE    The pool is initialized successfully.
1112   @retval FALSE   The memory is out of resource.
1113 **/
1114 BOOLEAN
1115 InitializePageTablePool (
1116   IN  UINTN  PoolPages
1117   )
1118 {
1119   VOID     *Buffer;
1120   BOOLEAN  IsModified;
1121
1122   //
1123   // Do not allow re-entrance.
1124   //
1125   if (mPageTablePoolLock) {
1126     return FALSE;
1127   }
1128
1129   mPageTablePoolLock = TRUE;
1130   IsModified         = FALSE;
1131
1132   //
1133   // Always reserve at least PAGE_TABLE_POOL_UNIT_PAGES, including one page for
1134   // header.
1135   //
1136   PoolPages += 1;   // Add one page for header.
1137   PoolPages  = ((PoolPages - 1) / PAGE_TABLE_POOL_UNIT_PAGES + 1) *
1138                PAGE_TABLE_POOL_UNIT_PAGES;
1139   Buffer = AllocateAlignedPages (PoolPages, PAGE_TABLE_POOL_ALIGNMENT);
1140   if (Buffer == NULL) {
1141     DEBUG ((DEBUG_ERROR, "ERROR: Out of aligned pages\r\n"));
1142     goto Done;
1143   }
1144
1145   DEBUG ((
1146     DEBUG_INFO,
1147     "Paging: added %lu pages to page table pool\r\n",
1148     (UINT64)PoolPages
1149     ));
1150
1151   //
1152   // Link all pools into a list for easier track later.
1153   //
1154   if (mPageTablePool == NULL) {
1155     mPageTablePool           = Buffer;
1156     mPageTablePool->NextPool = mPageTablePool;
1157   } else {
1158     ((PAGE_TABLE_POOL *)Buffer)->NextPool = mPageTablePool->NextPool;
1159     mPageTablePool->NextPool              = Buffer;
1160     mPageTablePool                        = Buffer;
1161   }
1162
1163   //
1164   // Reserve one page for pool header.
1165   //
1166   mPageTablePool->FreePages = PoolPages - 1;
1167   mPageTablePool->Offset    = EFI_PAGES_TO_SIZE (1);
1168
1169   //
1170   // Mark the whole pool pages as read-only.
1171   //
1172   ConvertMemoryPageAttributes (
1173     NULL,
1174     (PHYSICAL_ADDRESS)(UINTN)Buffer,
1175     EFI_PAGES_TO_SIZE (PoolPages),
1176     EFI_MEMORY_RO,
1177     PageActionSet,
1178     AllocatePageTableMemory,
1179     NULL,
1180     &IsModified
1181     );
1182   ASSERT (IsModified == TRUE);
1183
1184 Done:
1185   mPageTablePoolLock = FALSE;
1186   return IsModified;
1187 }
1188
1189 /**
1190   This API provides a way to allocate memory for page table.
1191
1192   This API can be called more than once to allocate memory for page tables.
1193
1194   Allocates the number of 4KB pages and returns a pointer to the allocated
1195   buffer. The buffer returned is aligned on a 4KB boundary.
1196
1197   If Pages is 0, then NULL is returned.
1198   If there is not enough memory remaining to satisfy the request, then NULL is
1199   returned.
1200
1201   @param  Pages                 The number of 4 KB pages to allocate.
1202
1203   @return A pointer to the allocated buffer or NULL if allocation fails.
1204
1205 **/
1206 VOID *
1207 EFIAPI
1208 AllocatePageTableMemory (
1209   IN UINTN  Pages
1210   )
1211 {
1212   VOID  *Buffer;
1213
1214   if (Pages == 0) {
1215     return NULL;
1216   }
1217
1218   //
1219   // Renew the pool if necessary.
1220   //
1221   if ((mPageTablePool == NULL) ||
1222       (Pages > mPageTablePool->FreePages))
1223   {
1224     if (!InitializePageTablePool (Pages)) {
1225       return NULL;
1226     }
1227   }
1228
1229   Buffer = (UINT8 *)mPageTablePool + mPageTablePool->Offset;
1230
1231   mPageTablePool->Offset    += EFI_PAGES_TO_SIZE (Pages);
1232   mPageTablePool->FreePages -= Pages;
1233
1234   return Buffer;
1235 }
1236
1237 /**
1238   Special handler for #DB exception, which will restore the page attributes
1239   (not-present). It should work with #PF handler which will set pages to
1240   'present'.
1241
1242   @param ExceptionType  Exception type.
1243   @param SystemContext  Pointer to EFI_SYSTEM_CONTEXT.
1244
1245 **/
1246 VOID
1247 EFIAPI
1248 DebugExceptionHandler (
1249   IN EFI_EXCEPTION_TYPE  ExceptionType,
1250   IN EFI_SYSTEM_CONTEXT  SystemContext
1251   )
1252 {
1253   UINTN    CpuIndex;
1254   UINTN    PFEntry;
1255   BOOLEAN  IsWpEnabled;
1256
1257   MpInitLibWhoAmI (&CpuIndex);
1258
1259   //
1260   // Clear last PF entries
1261   //
1262   IsWpEnabled = IsReadOnlyPageWriteProtected ();
1263   if (IsWpEnabled) {
1264     DisableReadOnlyPageWriteProtect ();
1265   }
1266
1267   for (PFEntry = 0; PFEntry < mPFEntryCount[CpuIndex]; PFEntry++) {
1268     if (mLastPFEntryPointer[CpuIndex][PFEntry] != NULL) {
1269       *mLastPFEntryPointer[CpuIndex][PFEntry] &= ~(UINT64)IA32_PG_P;
1270     }
1271   }
1272
1273   if (IsWpEnabled) {
1274     EnableReadOnlyPageWriteProtect ();
1275   }
1276
1277   //
1278   // Reset page fault exception count for next page fault.
1279   //
1280   mPFEntryCount[CpuIndex] = 0;
1281
1282   //
1283   // Flush TLB
1284   //
1285   CpuFlushTlb ();
1286
1287   //
1288   // Clear TF in EFLAGS
1289   //
1290   if (mPagingContext.MachineType == IMAGE_FILE_MACHINE_I386) {
1291     SystemContext.SystemContextIa32->Eflags &= (UINT32) ~BIT8;
1292   } else {
1293     SystemContext.SystemContextX64->Rflags &= (UINT64) ~BIT8;
1294   }
1295 }
1296
1297 /**
1298   Special handler for #PF exception, which will set the pages which caused
1299   #PF to be 'present'. The attribute of those pages should be restored in
1300   the subsequent #DB handler.
1301
1302   @param ExceptionType  Exception type.
1303   @param SystemContext  Pointer to EFI_SYSTEM_CONTEXT.
1304
1305 **/
1306 VOID
1307 EFIAPI
1308 PageFaultExceptionHandler (
1309   IN EFI_EXCEPTION_TYPE  ExceptionType,
1310   IN EFI_SYSTEM_CONTEXT  SystemContext
1311   )
1312 {
1313   EFI_STATUS                     Status;
1314   UINT64                         PFAddress;
1315   PAGE_TABLE_LIB_PAGING_CONTEXT  PagingContext;
1316   PAGE_ATTRIBUTE                 PageAttribute;
1317   UINT64                         Attributes;
1318   UINT64                         *PageEntry;
1319   UINTN                          Index;
1320   UINTN                          CpuIndex;
1321   UINTN                          PageNumber;
1322   BOOLEAN                        NonStopMode;
1323
1324   PFAddress = AsmReadCr2 () & ~EFI_PAGE_MASK;
1325   if (PFAddress < BASE_4KB) {
1326     NonStopMode = NULL_DETECTION_NONSTOP_MODE ? TRUE : FALSE;
1327   } else {
1328     NonStopMode = HEAP_GUARD_NONSTOP_MODE ? TRUE : FALSE;
1329   }
1330
1331   if (NonStopMode) {
1332     MpInitLibWhoAmI (&CpuIndex);
1333     GetCurrentPagingContext (&PagingContext);
1334     //
1335     // Memory operation cross page boundary, like "rep mov" instruction, will
1336     // cause infinite loop between this and Debug Trap handler. We have to make
1337     // sure that current page and the page followed are both in PRESENT state.
1338     //
1339     PageNumber = 2;
1340     while (PageNumber > 0) {
1341       PageEntry = GetPageTableEntry (&PagingContext, PFAddress, &PageAttribute);
1342       ASSERT (PageEntry != NULL);
1343
1344       if (PageEntry != NULL) {
1345         Attributes = GetAttributesFromPageEntry (PageEntry);
1346         if ((Attributes & EFI_MEMORY_RP) != 0) {
1347           Attributes &= ~EFI_MEMORY_RP;
1348           Status      = AssignMemoryPageAttributes (
1349                           &PagingContext,
1350                           PFAddress,
1351                           EFI_PAGE_SIZE,
1352                           Attributes,
1353                           NULL
1354                           );
1355           if (!EFI_ERROR (Status)) {
1356             Index = mPFEntryCount[CpuIndex];
1357             //
1358             // Re-retrieve page entry because above calling might update page
1359             // table due to table split.
1360             //
1361             PageEntry                              = GetPageTableEntry (&PagingContext, PFAddress, &PageAttribute);
1362             mLastPFEntryPointer[CpuIndex][Index++] = PageEntry;
1363             mPFEntryCount[CpuIndex]                = Index;
1364           }
1365         }
1366       }
1367
1368       PFAddress += EFI_PAGE_SIZE;
1369       --PageNumber;
1370     }
1371   }
1372
1373   //
1374   // Initialize the serial port before dumping.
1375   //
1376   SerialPortInitialize ();
1377   //
1378   // Display ExceptionType, CPU information and Image information
1379   //
1380   DumpCpuContext (ExceptionType, SystemContext);
1381   if (NonStopMode) {
1382     //
1383     // Set TF in EFLAGS
1384     //
1385     if (mPagingContext.MachineType == IMAGE_FILE_MACHINE_I386) {
1386       SystemContext.SystemContextIa32->Eflags |= (UINT32)BIT8;
1387     } else {
1388       SystemContext.SystemContextX64->Rflags |= (UINT64)BIT8;
1389     }
1390   } else {
1391     CpuDeadLoop ();
1392   }
1393 }
1394
1395 /**
1396   Initialize the Page Table lib.
1397 **/
1398 VOID
1399 InitializePageTableLib (
1400   VOID
1401   )
1402 {
1403   PAGE_TABLE_LIB_PAGING_CONTEXT  CurrentPagingContext;
1404   UINT32                         *Attributes;
1405   UINTN                          *PageTableBase;
1406
1407   GetCurrentPagingContext (&CurrentPagingContext);
1408
1409   GetPagingDetails (&CurrentPagingContext.ContextData, &PageTableBase, &Attributes);
1410
1411   //
1412   // Reserve memory of page tables for future uses, if paging is enabled.
1413   //
1414   if ((*PageTableBase != 0) &&
1415       ((*Attributes & PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE) != 0))
1416   {
1417     DisableReadOnlyPageWriteProtect ();
1418     InitializePageTablePool (1);
1419     EnableReadOnlyPageWriteProtect ();
1420   }
1421
1422   if (HEAP_GUARD_NONSTOP_MODE || NULL_DETECTION_NONSTOP_MODE) {
1423     mPFEntryCount = (UINTN *)AllocateZeroPool (sizeof (UINTN) * mNumberOfProcessors);
1424     ASSERT (mPFEntryCount != NULL);
1425
1426     mLastPFEntryPointer = (UINT64 *(*)[MAX_PF_ENTRY_COUNT])
1427                           AllocateZeroPool (sizeof (mLastPFEntryPointer[0]) * mNumberOfProcessors);
1428     ASSERT (mLastPFEntryPointer != NULL);
1429   }
1430
1431   DEBUG ((DEBUG_INFO, "CurrentPagingContext:\n"));
1432   DEBUG ((DEBUG_INFO, "  MachineType   - 0x%x\n", CurrentPagingContext.MachineType));
1433   DEBUG ((DEBUG_INFO, "  PageTableBase - 0x%Lx\n", (UINT64)*PageTableBase));
1434   DEBUG ((DEBUG_INFO, "  Attributes    - 0x%x\n", *Attributes));
1435
1436   return;
1437 }