UefiCpuPkg/CpuDxe/CpuPageTable.c

   1 /** @file
   2   Page table management support.
   3
   4   Copyright (c) 2017, Intel Corporation. All rights reserved.<BR>
   5   Copyright (c) 2017, AMD Incorporated. All rights reserved.<BR>
   6
   7   This program and the accompanying materials
   8   are licensed and made available under the terms and conditions of the BSD License
   9   which accompanies this distribution.  The full text of the license may be found at
  10   http://opensource.org/licenses/bsd-license.php
  11
  12   THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  13   WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  14
  15 **/
  16
  17 #include <Base.h>
  18 #include <Uefi.h>
  19 #include <Library/BaseLib.h>
  20 #include <Library/CpuLib.h>
  21 #include <Library/BaseMemoryLib.h>
  22 #include <Library/MemoryAllocationLib.h>
  23 #include <Library/DebugLib.h>
  24 #include <Library/UefiBootServicesTableLib.h>
  25 #include <Protocol/MpService.h>
  26 #include <Protocol/SmmBase2.h>
  27 #include <Register/Cpuid.h>
  28 #include <Register/Msr.h>
  29
  30 #include "CpuDxe.h"
  31 #include "CpuPageTable.h"
  32
  33 ///
  34 /// Paging registers
  35 ///
  36 #define CR0_WP                      BIT16
  37 #define CR0_PG                      BIT31
  38 #define CR4_PSE                     BIT4
  39 #define CR4_PAE                     BIT5
  40
  41 ///
  42 /// Page Table Entry
  43 ///
  44 #define IA32_PG_P                   BIT0
  45 #define IA32_PG_RW                  BIT1
  46 #define IA32_PG_U                   BIT2
  47 #define IA32_PG_WT                  BIT3
  48 #define IA32_PG_CD                  BIT4
  49 #define IA32_PG_A                   BIT5
  50 #define IA32_PG_D                   BIT6
  51 #define IA32_PG_PS                  BIT7
  52 #define IA32_PG_PAT_2M              BIT12
  53 #define IA32_PG_PAT_4K              IA32_PG_PS
  54 #define IA32_PG_PMNT                BIT62
  55 #define IA32_PG_NX                  BIT63
  56
  57 #define PAGE_ATTRIBUTE_BITS         (IA32_PG_D | IA32_PG_A | IA32_PG_U | IA32_PG_RW | IA32_PG_P)
  58 //
  59 // Bits 1, 2, 5, 6 are reserved in the IA32 PAE PDPTE
  60 // X64 PAE PDPTE does not have such restriction
  61 //
  62 #define IA32_PAE_PDPTE_ATTRIBUTE_BITS    (IA32_PG_P)
  63
  64 #define PAGE_PROGATE_BITS           (IA32_PG_NX | PAGE_ATTRIBUTE_BITS)
  65
  66 #define PAGING_4K_MASK  0xFFF
  67 #define PAGING_2M_MASK  0x1FFFFF
  68 #define PAGING_1G_MASK  0x3FFFFFFF
  69
  70 #define PAGING_PAE_INDEX_MASK  0x1FF
  71
  72 #define PAGING_4K_ADDRESS_MASK_64 0x000FFFFFFFFFF000ull
  73 #define PAGING_2M_ADDRESS_MASK_64 0x000FFFFFFFE00000ull
  74 #define PAGING_1G_ADDRESS_MASK_64 0x000FFFFFC0000000ull
  75
  76 typedef enum {
  77   PageNone,
  78   Page4K,
  79   Page2M,
  80   Page1G,
  81 } PAGE_ATTRIBUTE;
  82
  83 typedef struct {
  84   PAGE_ATTRIBUTE   Attribute;
  85   UINT64           Length;
  86   UINT64           AddressMask;
  87 } PAGE_ATTRIBUTE_TABLE;
  88
  89 typedef enum {
  90   PageActionAssign,
  91   PageActionSet,
  92   PageActionClear,
  93 } PAGE_ACTION;
  94
  95 PAGE_ATTRIBUTE_TABLE mPageAttributeTable[] = {
  96   {Page4K,  SIZE_4KB, PAGING_4K_ADDRESS_MASK_64},
  97   {Page2M,  SIZE_2MB, PAGING_2M_ADDRESS_MASK_64},
  98   {Page1G,  SIZE_1GB, PAGING_1G_ADDRESS_MASK_64},
  99 };
 100
 101 PAGE_TABLE_POOL                   *mPageTablePool = NULL;
 102 PAGE_TABLE_LIB_PAGING_CONTEXT     mPagingContext;
 103 EFI_SMM_BASE2_PROTOCOL            *mSmmBase2 = NULL;
 104
 105 /**
 106  Check if current execution environment is in SMM mode or not, via
 107  EFI_SMM_BASE2_PROTOCOL.
 108
 109  This is necessary because of the fact that MdePkg\Library\SmmMemoryAllocationLib
 110  supports to free memory outside SMRAM. The library will call gBS->FreePool() or
 111  gBS->FreePages() and then SetMemorySpaceAttributes interface in turn to change
 112  memory paging attributes during free operation, if some memory related features
 113  are enabled (like Heap Guard).
 114
 115  This means that SetMemorySpaceAttributes() has chance to run in SMM mode. This
 116  will cause incorrect result because SMM mode always loads its own page tables,
 117  which are usually different from DXE. This function can be used to detect such
 118  situation and help to avoid further misoperations.
 119
 120   @retval TRUE    In SMM mode.
 121   @retval FALSE   Not in SMM mode.
 122 **/
 123 BOOLEAN
 124 IsInSmm (
 125   VOID
 126   )
 127 {
 128   BOOLEAN                 InSmm;
 129
 130   InSmm = FALSE;
 131   if (mSmmBase2 == NULL) {
 132     gBS->LocateProtocol (&gEfiSmmBase2ProtocolGuid, NULL, (VOID **)&mSmmBase2);
 133   }
 134
 135   if (mSmmBase2 != NULL) {
 136     mSmmBase2->InSmm (mSmmBase2, &InSmm);
 137   }
 138
 139   return InSmm;
 140 }
 141
 142 /**
 143   Return current paging context.
 144
 145   @param[in,out]  PagingContext     The paging context.
 146 **/
 147 VOID
 148 GetCurrentPagingContext (
 149   IN OUT PAGE_TABLE_LIB_PAGING_CONTEXT     *PagingContext
 150   )
 151 {
 152   UINT32                          RegEax;
 153   CPUID_EXTENDED_CPU_SIG_EDX      RegEdx;
 154   MSR_IA32_EFER_REGISTER          MsrEfer;
 155
 156   //
 157   // Don't retrieve current paging context from processor if in SMM mode.
 158   //
 159   if (!IsInSmm ()) {
 160     ZeroMem (&mPagingContext, sizeof(mPagingContext));
 161     if (sizeof(UINTN) == sizeof(UINT64)) {
 162       mPagingContext.MachineType = IMAGE_FILE_MACHINE_X64;
 163     } else {
 164       mPagingContext.MachineType = IMAGE_FILE_MACHINE_I386;
 165     }
 166     if ((AsmReadCr0 () & CR0_PG) != 0) {
 167       mPagingContext.ContextData.X64.PageTableBase = (AsmReadCr3 () & PAGING_4K_ADDRESS_MASK_64);
 168     } else {
 169       mPagingContext.ContextData.X64.PageTableBase = 0;
 170     }
 171
 172     if ((AsmReadCr4 () & CR4_PSE) != 0) {
 173       mPagingContext.ContextData.Ia32.Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PSE;
 174     }
 175     if ((AsmReadCr4 () & CR4_PAE) != 0) {
 176       mPagingContext.ContextData.Ia32.Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE;
 177     }
 178     if ((AsmReadCr0 () & CR0_WP) != 0) {
 179       mPagingContext.ContextData.Ia32.Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_WP_ENABLE;
 180     }
 181
 182     AsmCpuid (CPUID_EXTENDED_FUNCTION, &RegEax, NULL, NULL, NULL);
 183     if (RegEax >= CPUID_EXTENDED_CPU_SIG) {
 184       AsmCpuid (CPUID_EXTENDED_CPU_SIG, NULL, NULL, NULL, &RegEdx.Uint32);
 185
 186       if (RegEdx.Bits.NX != 0) {
 187         // XD supported
 188         MsrEfer.Uint64 = AsmReadMsr64(MSR_CORE_IA32_EFER);
 189         if (MsrEfer.Bits.NXE != 0) {
 190           // XD activated
 191           mPagingContext.ContextData.Ia32.Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_XD_ACTIVATED;
 192         }
 193       }
 194
 195       if (RegEdx.Bits.Page1GB != 0) {
 196         mPagingContext.ContextData.Ia32.Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAGE_1G_SUPPORT;
 197       }
 198     }
 199   }
 200
 201   //
 202   // This can avoid getting SMM paging context if in SMM mode. We cannot assume
 203   // SMM mode shares the same paging context as DXE.
 204   //
 205   CopyMem (PagingContext, &mPagingContext, sizeof (mPagingContext));
 206 }
 207
 208 /**
 209   Return length according to page attributes.
 210
 211   @param[in]  PageAttributes   The page attribute of the page entry.
 212
 213   @return The length of page entry.
 214 **/
 215 UINTN
 216 PageAttributeToLength (
 217   IN PAGE_ATTRIBUTE  PageAttribute
 218   )
 219 {
 220   UINTN  Index;
 221   for (Index = 0; Index < sizeof(mPageAttributeTable)/sizeof(mPageAttributeTable[0]); Index++) {
 222     if (PageAttribute == mPageAttributeTable[Index].Attribute) {
 223       return (UINTN)mPageAttributeTable[Index].Length;
 224     }
 225   }
 226   return 0;
 227 }
 228
 229 /**
 230   Return address mask according to page attributes.
 231
 232   @param[in]  PageAttributes   The page attribute of the page entry.
 233
 234   @return The address mask of page entry.
 235 **/
 236 UINTN
 237 PageAttributeToMask (
 238   IN PAGE_ATTRIBUTE  PageAttribute
 239   )
 240 {
 241   UINTN  Index;
 242   for (Index = 0; Index < sizeof(mPageAttributeTable)/sizeof(mPageAttributeTable[0]); Index++) {
 243     if (PageAttribute == mPageAttributeTable[Index].Attribute) {
 244       return (UINTN)mPageAttributeTable[Index].AddressMask;
 245     }
 246   }
 247   return 0;
 248 }
 249
 250 /**
 251   Return page table entry to match the address.
 252
 253   @param[in]  PagingContext     The paging context.
 254   @param[in]  Address           The address to be checked.
 255   @param[out] PageAttributes    The page attribute of the page entry.
 256
 257   @return The page entry.
 258 **/
 259 VOID *
 260 GetPageTableEntry (
 261   IN  PAGE_TABLE_LIB_PAGING_CONTEXT     *PagingContext,
 262   IN  PHYSICAL_ADDRESS                  Address,
 263   OUT PAGE_ATTRIBUTE                    *PageAttribute
 264   )
 265 {
 266   UINTN                 Index1;
 267   UINTN                 Index2;
 268   UINTN                 Index3;
 269   UINTN                 Index4;
 270   UINT64                *L1PageTable;
 271   UINT64                *L2PageTable;
 272   UINT64                *L3PageTable;
 273   UINT64                *L4PageTable;
 274   UINT64                AddressEncMask;
 275
 276   ASSERT (PagingContext != NULL);
 277
 278   Index4 = ((UINTN)RShiftU64 (Address, 39)) & PAGING_PAE_INDEX_MASK;
 279   Index3 = ((UINTN)Address >> 30) & PAGING_PAE_INDEX_MASK;
 280   Index2 = ((UINTN)Address >> 21) & PAGING_PAE_INDEX_MASK;
 281   Index1 = ((UINTN)Address >> 12) & PAGING_PAE_INDEX_MASK;
 282
 283   // Make sure AddressEncMask is contained to smallest supported address field.
 284   //
 285   AddressEncMask = PcdGet64 (PcdPteMemoryEncryptionAddressOrMask) & PAGING_1G_ADDRESS_MASK_64;
 286
 287   if (PagingContext->MachineType == IMAGE_FILE_MACHINE_X64) {
 288     L4PageTable = (UINT64 *)(UINTN)PagingContext->ContextData.X64.PageTableBase;
 289     if (L4PageTable[Index4] == 0) {
 290       *PageAttribute = PageNone;
 291       return NULL;
 292     }
 293
 294     L3PageTable = (UINT64 *)(UINTN)(L4PageTable[Index4] & ~AddressEncMask & PAGING_4K_ADDRESS_MASK_64);
 295   } else {
 296     ASSERT((PagingContext->ContextData.Ia32.Attributes & PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE) != 0);
 297     L3PageTable = (UINT64 *)(UINTN)PagingContext->ContextData.Ia32.PageTableBase;
 298   }
 299   if (L3PageTable[Index3] == 0) {
 300     *PageAttribute = PageNone;
 301     return NULL;
 302   }
 303   if ((L3PageTable[Index3] & IA32_PG_PS) != 0) {
 304     // 1G
 305     *PageAttribute = Page1G;
 306     return &L3PageTable[Index3];
 307   }
 308
 309   L2PageTable = (UINT64 *)(UINTN)(L3PageTable[Index3] & ~AddressEncMask & PAGING_4K_ADDRESS_MASK_64);
 310   if (L2PageTable[Index2] == 0) {
 311     *PageAttribute = PageNone;
 312     return NULL;
 313   }
 314   if ((L2PageTable[Index2] & IA32_PG_PS) != 0) {
 315     // 2M
 316     *PageAttribute = Page2M;
 317     return &L2PageTable[Index2];
 318   }
 319
 320   // 4k
 321   L1PageTable = (UINT64 *)(UINTN)(L2PageTable[Index2] & ~AddressEncMask & PAGING_4K_ADDRESS_MASK_64);
 322   if ((L1PageTable[Index1] == 0) && (Address != 0)) {
 323     *PageAttribute = PageNone;
 324     return NULL;
 325   }
 326   *PageAttribute = Page4K;
 327   return &L1PageTable[Index1];
 328 }
 329
 330 /**
 331   Return memory attributes of page entry.
 332
 333   @param[in]  PageEntry        The page entry.
 334
 335   @return Memory attributes of page entry.
 336 **/
 337 UINT64
 338 GetAttributesFromPageEntry (
 339   IN  UINT64                            *PageEntry
 340   )
 341 {
 342   UINT64  Attributes;
 343   Attributes = 0;
 344   if ((*PageEntry & IA32_PG_P) == 0) {
 345     Attributes |= EFI_MEMORY_RP;
 346   }
 347   if ((*PageEntry & IA32_PG_RW) == 0) {
 348     Attributes |= EFI_MEMORY_RO;
 349   }
 350   if ((*PageEntry & IA32_PG_NX) != 0) {
 351     Attributes |= EFI_MEMORY_XP;
 352   }
 353   return Attributes;
 354 }
 355
 356 /**
 357   Modify memory attributes of page entry.
 358
 359   @param[in]  PagingContext    The paging context.
 360   @param[in]  PageEntry        The page entry.
 361   @param[in]  Attributes       The bit mask of attributes to modify for the memory region.
 362   @param[in]  PageAction       The page action.
 363   @param[out] IsModified       TRUE means page table modified. FALSE means page table not modified.
 364 **/
 365 VOID
 366 ConvertPageEntryAttribute (
 367   IN  PAGE_TABLE_LIB_PAGING_CONTEXT     *PagingContext,
 368   IN  UINT64                            *PageEntry,
 369   IN  UINT64                            Attributes,
 370   IN  PAGE_ACTION                       PageAction,
 371   OUT BOOLEAN                           *IsModified
 372   )
 373 {
 374   UINT64  CurrentPageEntry;
 375   UINT64  NewPageEntry;
 376
 377   CurrentPageEntry = *PageEntry;
 378   NewPageEntry = CurrentPageEntry;
 379   if ((Attributes & EFI_MEMORY_RP) != 0) {
 380     switch (PageAction) {
 381     case PageActionAssign:
 382     case PageActionSet:
 383       NewPageEntry &= ~(UINT64)IA32_PG_P;
 384       break;
 385     case PageActionClear:
 386       NewPageEntry |= IA32_PG_P;
 387       break;
 388     }
 389   } else {
 390     switch (PageAction) {
 391     case PageActionAssign:
 392       NewPageEntry |= IA32_PG_P;
 393       break;
 394     case PageActionSet:
 395     case PageActionClear:
 396       break;
 397     }
 398   }
 399   if ((Attributes & EFI_MEMORY_RO) != 0) {
 400     switch (PageAction) {
 401     case PageActionAssign:
 402     case PageActionSet:
 403       NewPageEntry &= ~(UINT64)IA32_PG_RW;
 404       break;
 405     case PageActionClear:
 406       NewPageEntry |= IA32_PG_RW;
 407       break;
 408     }
 409   } else {
 410     switch (PageAction) {
 411     case PageActionAssign:
 412       NewPageEntry |= IA32_PG_RW;
 413       break;
 414     case PageActionSet:
 415     case PageActionClear:
 416       break;
 417     }
 418   }
 419   if ((PagingContext->ContextData.Ia32.Attributes & PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_XD_ACTIVATED) != 0) {
 420     if ((Attributes & EFI_MEMORY_XP) != 0) {
 421       switch (PageAction) {
 422       case PageActionAssign:
 423       case PageActionSet:
 424         NewPageEntry |= IA32_PG_NX;
 425         break;
 426       case PageActionClear:
 427         NewPageEntry &= ~IA32_PG_NX;
 428         break;
 429       }
 430     } else {
 431       switch (PageAction) {
 432       case PageActionAssign:
 433         NewPageEntry &= ~IA32_PG_NX;
 434         break;
 435       case PageActionSet:
 436       case PageActionClear:
 437         break;
 438       }
 439     }
 440   }
 441   *PageEntry = NewPageEntry;
 442   if (CurrentPageEntry != NewPageEntry) {
 443     *IsModified = TRUE;
 444     DEBUG ((DEBUG_VERBOSE, "ConvertPageEntryAttribute 0x%lx", CurrentPageEntry));
 445     DEBUG ((DEBUG_VERBOSE, "->0x%lx\n", NewPageEntry));
 446   } else {
 447     *IsModified = FALSE;
 448   }
 449 }
 450
 451 /**
 452   This function returns if there is need to split page entry.
 453
 454   @param[in]  BaseAddress      The base address to be checked.
 455   @param[in]  Length           The length to be checked.
 456   @param[in]  PageEntry        The page entry to be checked.
 457   @param[in]  PageAttribute    The page attribute of the page entry.
 458
 459   @retval SplitAttributes on if there is need to split page entry.
 460 **/
 461 PAGE_ATTRIBUTE
 462 NeedSplitPage (
 463   IN  PHYSICAL_ADDRESS                  BaseAddress,
 464   IN  UINT64                            Length,
 465   IN  UINT64                            *PageEntry,
 466   IN  PAGE_ATTRIBUTE                    PageAttribute
 467   )
 468 {
 469   UINT64                PageEntryLength;
 470
 471   PageEntryLength = PageAttributeToLength (PageAttribute);
 472
 473   if (((BaseAddress & (PageEntryLength - 1)) == 0) && (Length >= PageEntryLength)) {
 474     return PageNone;
 475   }
 476
 477   if (((BaseAddress & PAGING_2M_MASK) != 0) || (Length < SIZE_2MB)) {
 478     return Page4K;
 479   }
 480
 481   return Page2M;
 482 }
 483
 484 /**
 485   This function splits one page entry to small page entries.
 486
 487   @param[in]  PageEntry         The page entry to be splitted.
 488   @param[in]  PageAttribute     The page attribute of the page entry.
 489   @param[in]  SplitAttribute    How to split the page entry.
 490   @param[in]  AllocatePagesFunc If page split is needed, this function is used to allocate more pages.
 491
 492   @retval RETURN_SUCCESS            The page entry is splitted.
 493   @retval RETURN_UNSUPPORTED        The page entry does not support to be splitted.
 494   @retval RETURN_OUT_OF_RESOURCES   No resource to split page entry.
 495 **/
 496 RETURN_STATUS
 497 SplitPage (
 498   IN  UINT64                            *PageEntry,
 499   IN  PAGE_ATTRIBUTE                    PageAttribute,
 500   IN  PAGE_ATTRIBUTE                    SplitAttribute,
 501   IN  PAGE_TABLE_LIB_ALLOCATE_PAGES     AllocatePagesFunc
 502   )
 503 {
 504   UINT64   BaseAddress;
 505   UINT64   *NewPageEntry;
 506   UINTN    Index;
 507   UINT64   AddressEncMask;
 508
 509   ASSERT (PageAttribute == Page2M || PageAttribute == Page1G);
 510
 511   ASSERT (AllocatePagesFunc != NULL);
 512
 513   // Make sure AddressEncMask is contained to smallest supported address field.
 514   //
 515   AddressEncMask = PcdGet64 (PcdPteMemoryEncryptionAddressOrMask) & PAGING_1G_ADDRESS_MASK_64;
 516
 517   if (PageAttribute == Page2M) {
 518     //
 519     // Split 2M to 4K
 520     //
 521     ASSERT (SplitAttribute == Page4K);
 522     if (SplitAttribute == Page4K) {
 523       NewPageEntry = AllocatePagesFunc (1);
 524       DEBUG ((DEBUG_INFO, "Split - 0x%x\n", NewPageEntry));
 525       if (NewPageEntry == NULL) {
 526         return RETURN_OUT_OF_RESOURCES;
 527       }
 528       BaseAddress = *PageEntry & ~AddressEncMask & PAGING_2M_ADDRESS_MASK_64;
 529       for (Index = 0; Index < SIZE_4KB / sizeof(UINT64); Index++) {
 530         NewPageEntry[Index] = (BaseAddress + SIZE_4KB * Index) | AddressEncMask | ((*PageEntry) & PAGE_PROGATE_BITS);
 531       }
 532       (*PageEntry) = (UINT64)(UINTN)NewPageEntry | AddressEncMask | ((*PageEntry) & PAGE_ATTRIBUTE_BITS);
 533       return RETURN_SUCCESS;
 534     } else {
 535       return RETURN_UNSUPPORTED;
 536     }
 537   } else if (PageAttribute == Page1G) {
 538     //
 539     // Split 1G to 2M
 540     // No need support 1G->4K directly, we should use 1G->2M, then 2M->4K to get more compact page table.
 541     //
 542     ASSERT (SplitAttribute == Page2M || SplitAttribute == Page4K);
 543     if ((SplitAttribute == Page2M || SplitAttribute == Page4K)) {
 544       NewPageEntry = AllocatePagesFunc (1);
 545       DEBUG ((DEBUG_INFO, "Split - 0x%x\n", NewPageEntry));
 546       if (NewPageEntry == NULL) {
 547         return RETURN_OUT_OF_RESOURCES;
 548       }
 549       BaseAddress = *PageEntry & ~AddressEncMask  & PAGING_1G_ADDRESS_MASK_64;
 550       for (Index = 0; Index < SIZE_4KB / sizeof(UINT64); Index++) {
 551         NewPageEntry[Index] = (BaseAddress + SIZE_2MB * Index) | AddressEncMask | IA32_PG_PS | ((*PageEntry) & PAGE_PROGATE_BITS);
 552       }
 553       (*PageEntry) = (UINT64)(UINTN)NewPageEntry | AddressEncMask | ((*PageEntry) & PAGE_ATTRIBUTE_BITS);
 554       return RETURN_SUCCESS;
 555     } else {
 556       return RETURN_UNSUPPORTED;
 557     }
 558   } else {
 559     return RETURN_UNSUPPORTED;
 560   }
 561 }
 562
 563 /**
 564  Check the WP status in CR0 register. This bit is used to lock or unlock write
 565  access to pages marked as read-only.
 566
 567   @retval TRUE    Write protection is enabled.
 568   @retval FALSE   Write protection is disabled.
 569 **/
 570 BOOLEAN
 571 IsReadOnlyPageWriteProtected (
 572   VOID
 573   )
 574 {
 575   //
 576   // To avoid unforseen consequences, don't touch paging settings in SMM mode
 577   // in this driver.
 578   //
 579   if (!IsInSmm ()) {
 580     return ((AsmReadCr0 () & CR0_WP) != 0);
 581   }
 582   return FALSE;
 583 }
 584
 585 /**
 586  Disable Write Protect on pages marked as read-only.
 587 **/
 588 VOID
 589 DisableReadOnlyPageWriteProtect (
 590   VOID
 591   )
 592 {
 593   //
 594   // To avoid unforseen consequences, don't touch paging settings in SMM mode
 595   // in this driver.
 596   //
 597   if (!IsInSmm ()) {
 598     AsmWriteCr0 (AsmReadCr0 () & ~CR0_WP);
 599   }
 600 }
 601
 602 /**
 603  Enable Write Protect on pages marked as read-only.
 604 **/
 605 VOID
 606 EnableReadOnlyPageWriteProtect (
 607   VOID
 608   )
 609 {
 610   //
 611   // To avoid unforseen consequences, don't touch paging settings in SMM mode
 612   // in this driver.
 613   //
 614   if (!IsInSmm ()) {
 615     AsmWriteCr0 (AsmReadCr0 () | CR0_WP);
 616   }
 617 }
 618
 619 /**
 620   This function modifies the page attributes for the memory region specified by BaseAddress and
 621   Length from their current attributes to the attributes specified by Attributes.
 622
 623   Caller should make sure BaseAddress and Length is at page boundary.
 624
 625   @param[in]  PagingContext     The paging context. NULL means get page table from current CPU context.
 626   @param[in]  BaseAddress       The physical address that is the start address of a memory region.
 627   @param[in]  Length            The size in bytes of the memory region.
 628   @param[in]  Attributes        The bit mask of attributes to modify for the memory region.
 629   @param[in]  PageAction        The page action.
 630   @param[in]  AllocatePagesFunc If page split is needed, this function is used to allocate more pages.
 631                                 NULL mean page split is unsupported.
 632   @param[out] IsSplitted        TRUE means page table splitted. FALSE means page table not splitted.
 633   @param[out] IsModified        TRUE means page table modified. FALSE means page table not modified.
 634
 635   @retval RETURN_SUCCESS           The attributes were modified for the memory region.
 636   @retval RETURN_ACCESS_DENIED     The attributes for the memory resource range specified by
 637                                    BaseAddress and Length cannot be modified.
 638   @retval RETURN_INVALID_PARAMETER Length is zero.
 639                                    Attributes specified an illegal combination of attributes that
 640                                    cannot be set together.
 641   @retval RETURN_OUT_OF_RESOURCES  There are not enough system resources to modify the attributes of
 642                                    the memory resource range.
 643   @retval RETURN_UNSUPPORTED       The processor does not support one or more bytes of the memory
 644                                    resource range specified by BaseAddress and Length.
 645                                    The bit mask of attributes is not support for the memory resource
 646                                    range specified by BaseAddress and Length.
 647 **/
 648 RETURN_STATUS
 649 ConvertMemoryPageAttributes (
 650   IN  PAGE_TABLE_LIB_PAGING_CONTEXT     *PagingContext OPTIONAL,
 651   IN  PHYSICAL_ADDRESS                  BaseAddress,
 652   IN  UINT64                            Length,
 653   IN  UINT64                            Attributes,
 654   IN  PAGE_ACTION                       PageAction,
 655   IN  PAGE_TABLE_LIB_ALLOCATE_PAGES     AllocatePagesFunc OPTIONAL,
 656   OUT BOOLEAN                           *IsSplitted,  OPTIONAL
 657   OUT BOOLEAN                           *IsModified   OPTIONAL
 658   )
 659 {
 660   PAGE_TABLE_LIB_PAGING_CONTEXT     CurrentPagingContext;
 661   UINT64                            *PageEntry;
 662   PAGE_ATTRIBUTE                    PageAttribute;
 663   UINTN                             PageEntryLength;
 664   PAGE_ATTRIBUTE                    SplitAttribute;
 665   RETURN_STATUS                     Status;
 666   BOOLEAN                           IsEntryModified;
 667   BOOLEAN                           IsWpEnabled;
 668
 669   if ((BaseAddress & (SIZE_4KB - 1)) != 0) {
 670     DEBUG ((DEBUG_ERROR, "BaseAddress(0x%lx) is not aligned!\n", BaseAddress));
 671     return EFI_UNSUPPORTED;
 672   }
 673   if ((Length & (SIZE_4KB - 1)) != 0) {
 674     DEBUG ((DEBUG_ERROR, "Length(0x%lx) is not aligned!\n", Length));
 675     return EFI_UNSUPPORTED;
 676   }
 677   if (Length == 0) {
 678     DEBUG ((DEBUG_ERROR, "Length is 0!\n"));
 679     return RETURN_INVALID_PARAMETER;
 680   }
 681
 682   if ((Attributes & ~(EFI_MEMORY_RP | EFI_MEMORY_RO | EFI_MEMORY_XP)) != 0) {
 683     DEBUG ((DEBUG_ERROR, "Attributes(0x%lx) has unsupported bit\n", Attributes));
 684     return EFI_UNSUPPORTED;
 685   }
 686
 687   if (PagingContext == NULL) {
 688     GetCurrentPagingContext (&CurrentPagingContext);
 689   } else {
 690     CopyMem (&CurrentPagingContext, PagingContext, sizeof(CurrentPagingContext));
 691   }
 692   switch(CurrentPagingContext.MachineType) {
 693   case IMAGE_FILE_MACHINE_I386:
 694     if (CurrentPagingContext.ContextData.Ia32.PageTableBase == 0) {
 695       if (Attributes == 0) {
 696         return EFI_SUCCESS;
 697       } else {
 698         DEBUG ((DEBUG_ERROR, "PageTable is 0!\n"));
 699         return EFI_UNSUPPORTED;
 700       }
 701     }
 702     if ((CurrentPagingContext.ContextData.Ia32.Attributes & PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE) == 0) {
 703       DEBUG ((DEBUG_ERROR, "Non-PAE Paging!\n"));
 704       return EFI_UNSUPPORTED;
 705     }
 706     if ((BaseAddress + Length) > BASE_4GB) {
 707       DEBUG ((DEBUG_ERROR, "Beyond 4GB memory in 32-bit mode!\n"));
 708       return EFI_UNSUPPORTED;
 709     }
 710     break;
 711   case IMAGE_FILE_MACHINE_X64:
 712     ASSERT (CurrentPagingContext.ContextData.X64.PageTableBase != 0);
 713     break;
 714   default:
 715     ASSERT(FALSE);
 716     return EFI_UNSUPPORTED;
 717     break;
 718   }
 719
 720 //  DEBUG ((DEBUG_ERROR, "ConvertMemoryPageAttributes(%x) - %016lx, %016lx, %02lx\n", IsSet, BaseAddress, Length, Attributes));
 721
 722   if (IsSplitted != NULL) {
 723     *IsSplitted = FALSE;
 724   }
 725   if (IsModified != NULL) {
 726     *IsModified = FALSE;
 727   }
 728   if (AllocatePagesFunc == NULL) {
 729     AllocatePagesFunc = AllocatePageTableMemory;
 730   }
 731
 732   //
 733   // Make sure that the page table is changeable.
 734   //
 735   IsWpEnabled = IsReadOnlyPageWriteProtected ();
 736   if (IsWpEnabled) {
 737     DisableReadOnlyPageWriteProtect ();
 738   }
 739
 740   //
 741   // Below logic is to check 2M/4K page to make sure we donot waist memory.
 742   //
 743   Status = EFI_SUCCESS;
 744   while (Length != 0) {
 745     PageEntry = GetPageTableEntry (&CurrentPagingContext, BaseAddress, &PageAttribute);
 746     if (PageEntry == NULL) {
 747       Status = RETURN_UNSUPPORTED;
 748       goto Done;
 749     }
 750     PageEntryLength = PageAttributeToLength (PageAttribute);
 751     SplitAttribute = NeedSplitPage (BaseAddress, Length, PageEntry, PageAttribute);
 752     if (SplitAttribute == PageNone) {
 753       ConvertPageEntryAttribute (&CurrentPagingContext, PageEntry, Attributes, PageAction, &IsEntryModified);
 754       if (IsEntryModified) {
 755         if (IsModified != NULL) {
 756           *IsModified = TRUE;
 757         }
 758       }
 759       //
 760       // Convert success, move to next
 761       //
 762       BaseAddress += PageEntryLength;
 763       Length -= PageEntryLength;
 764     } else {
 765       if (AllocatePagesFunc == NULL) {
 766         Status = RETURN_UNSUPPORTED;
 767         goto Done;
 768       }
 769       Status = SplitPage (PageEntry, PageAttribute, SplitAttribute, AllocatePagesFunc);
 770       if (RETURN_ERROR (Status)) {
 771         Status = RETURN_UNSUPPORTED;
 772         goto Done;
 773       }
 774       if (IsSplitted != NULL) {
 775         *IsSplitted = TRUE;
 776       }
 777       if (IsModified != NULL) {
 778         *IsModified = TRUE;
 779       }
 780       //
 781       // Just split current page
 782       // Convert success in next around
 783       //
 784     }
 785   }
 786
 787 Done:
 788   //
 789   // Restore page table write protection, if any.
 790   //
 791   if (IsWpEnabled) {
 792     EnableReadOnlyPageWriteProtect ();
 793   }
 794   return Status;
 795 }
 796
 797 /**
 798   This function assigns the page attributes for the memory region specified by BaseAddress and
 799   Length from their current attributes to the attributes specified by Attributes.
 800
 801   Caller should make sure BaseAddress and Length is at page boundary.
 802
 803   Caller need guarentee the TPL <= TPL_NOTIFY, if there is split page request.
 804
 805   @param[in]  PagingContext     The paging context. NULL means get page table from current CPU context.
 806   @param[in]  BaseAddress       The physical address that is the start address of a memory region.
 807   @param[in]  Length            The size in bytes of the memory region.
 808   @param[in]  Attributes        The bit mask of attributes to set for the memory region.
 809   @param[in]  AllocatePagesFunc If page split is needed, this function is used to allocate more pages.
 810                                 NULL mean page split is unsupported.
 811
 812   @retval RETURN_SUCCESS           The attributes were cleared for the memory region.
 813   @retval RETURN_ACCESS_DENIED     The attributes for the memory resource range specified by
 814                                    BaseAddress and Length cannot be modified.
 815   @retval RETURN_INVALID_PARAMETER Length is zero.
 816                                    Attributes specified an illegal combination of attributes that
 817                                    cannot be set together.
 818   @retval RETURN_OUT_OF_RESOURCES  There are not enough system resources to modify the attributes of
 819                                    the memory resource range.
 820   @retval RETURN_UNSUPPORTED       The processor does not support one or more bytes of the memory
 821                                    resource range specified by BaseAddress and Length.
 822                                    The bit mask of attributes is not support for the memory resource
 823                                    range specified by BaseAddress and Length.
 824 **/
 825 RETURN_STATUS
 826 EFIAPI
 827 AssignMemoryPageAttributes (
 828   IN  PAGE_TABLE_LIB_PAGING_CONTEXT     *PagingContext OPTIONAL,
 829   IN  PHYSICAL_ADDRESS                  BaseAddress,
 830   IN  UINT64                            Length,
 831   IN  UINT64                            Attributes,
 832   IN  PAGE_TABLE_LIB_ALLOCATE_PAGES     AllocatePagesFunc OPTIONAL
 833   )
 834 {
 835   RETURN_STATUS  Status;
 836   BOOLEAN        IsModified;
 837   BOOLEAN        IsSplitted;
 838
 839 //  DEBUG((DEBUG_INFO, "AssignMemoryPageAttributes: 0x%lx - 0x%lx (0x%lx)\n", BaseAddress, Length, Attributes));
 840   Status = ConvertMemoryPageAttributes (PagingContext, BaseAddress, Length, Attributes, PageActionAssign, AllocatePagesFunc, &IsSplitted, &IsModified);
 841   if (!EFI_ERROR(Status)) {
 842     if ((PagingContext == NULL) && IsModified) {
 843       //
 844       // Flush TLB as last step.
 845       //
 846       // Note: Since APs will always init CR3 register in HLT loop mode or do
 847       // TLB flush in MWAIT loop mode, there's no need to flush TLB for them
 848       // here.
 849       //
 850       CpuFlushTlb();
 851     }
 852   }
 853
 854   return Status;
 855 }
 856
 857 /**
 858  Check if Execute Disable feature is enabled or not.
 859 **/
 860 BOOLEAN
 861 IsExecuteDisableEnabled (
 862   VOID
 863   )
 864 {
 865   MSR_CORE_IA32_EFER_REGISTER    MsrEfer;
 866
 867   MsrEfer.Uint64 = AsmReadMsr64 (MSR_IA32_EFER);
 868   return (MsrEfer.Bits.NXE == 1);
 869 }
 870
 871 /**
 872   Update GCD memory space attributes according to current page table setup.
 873 **/
 874 VOID
 875 RefreshGcdMemoryAttributesFromPaging (
 876   VOID
 877   )
 878 {
 879   EFI_STATUS                          Status;
 880   UINTN                               NumberOfDescriptors;
 881   EFI_GCD_MEMORY_SPACE_DESCRIPTOR     *MemorySpaceMap;
 882   PAGE_TABLE_LIB_PAGING_CONTEXT       PagingContext;
 883   PAGE_ATTRIBUTE                      PageAttribute;
 884   UINT64                              *PageEntry;
 885   UINT64                              PageLength;
 886   UINT64                              MemorySpaceLength;
 887   UINT64                              Length;
 888   UINT64                              BaseAddress;
 889   UINT64                              PageStartAddress;
 890   UINT64                              Attributes;
 891   UINT64                              Capabilities;
 892   UINT64                              NewAttributes;
 893   UINTN                               Index;
 894
 895   //
 896   // Assuming that memory space map returned is sorted already; otherwise sort
 897   // them in the order of lowest address to highest address.
 898   //
 899   Status = gDS->GetMemorySpaceMap (&NumberOfDescriptors, &MemorySpaceMap);
 900   ASSERT_EFI_ERROR (Status);
 901
 902   GetCurrentPagingContext (&PagingContext);
 903
 904   Attributes      = 0;
 905   NewAttributes   = 0;
 906   BaseAddress     = 0;
 907   PageLength      = 0;
 908
 909   if (IsExecuteDisableEnabled ()) {
 910     Capabilities = EFI_MEMORY_RO | EFI_MEMORY_RP | EFI_MEMORY_XP;
 911   } else {
 912     Capabilities = EFI_MEMORY_RO | EFI_MEMORY_RP;
 913   }
 914
 915   for (Index = 0; Index < NumberOfDescriptors; Index++) {
 916     if (MemorySpaceMap[Index].GcdMemoryType == EfiGcdMemoryTypeNonExistent) {
 917       continue;
 918     }
 919
 920     //
 921     // Sync the actual paging related capabilities back to GCD service first.
 922     // As a side effect (good one), this can also help to avoid unnecessary
 923     // memory map entries due to the different capabilities of the same type
 924     // memory, such as multiple RT_CODE and RT_DATA entries in memory map,
 925     // which could cause boot failure of some old Linux distro (before v4.3).
 926     //
 927     Status = gDS->SetMemorySpaceCapabilities (
 928                     MemorySpaceMap[Index].BaseAddress,
 929                     MemorySpaceMap[Index].Length,
 930                     MemorySpaceMap[Index].Capabilities | Capabilities
 931                     );
 932     if (EFI_ERROR (Status)) {
 933       //
 934       // If we cannot udpate the capabilities, we cannot update its
 935       // attributes either. So just simply skip current block of memory.
 936       //
 937       DEBUG ((
 938         DEBUG_WARN,
 939         "Failed to update capability: [%lu] %016lx - %016lx (%016lx -> %016lx)\r\n",
 940         (UINT64)Index, MemorySpaceMap[Index].BaseAddress,
 941         MemorySpaceMap[Index].BaseAddress + MemorySpaceMap[Index].Length - 1,
 942         MemorySpaceMap[Index].Capabilities,
 943         MemorySpaceMap[Index].Capabilities | Capabilities
 944         ));
 945       continue;
 946     }
 947
 948     if (MemorySpaceMap[Index].BaseAddress >= (BaseAddress + PageLength)) {
 949       //
 950       // Current memory space starts at a new page. Resetting PageLength will
 951       // trigger a retrieval of page attributes at new address.
 952       //
 953       PageLength = 0;
 954     } else {
 955       //
 956       // In case current memory space is not adjacent to last one
 957       //
 958       PageLength -= (MemorySpaceMap[Index].BaseAddress - BaseAddress);
 959     }
 960
 961     //
 962     // Sync actual page attributes to GCD
 963     //
 964     BaseAddress       = MemorySpaceMap[Index].BaseAddress;
 965     MemorySpaceLength = MemorySpaceMap[Index].Length;
 966     while (MemorySpaceLength > 0) {
 967       if (PageLength == 0) {
 968         PageEntry = GetPageTableEntry (&PagingContext, BaseAddress, &PageAttribute);
 969         if (PageEntry == NULL) {
 970           break;
 971         }
 972
 973         //
 974         // Note current memory space might start in the middle of a page
 975         //
 976         PageStartAddress  = (*PageEntry) & (UINT64)PageAttributeToMask(PageAttribute);
 977         PageLength        = PageAttributeToLength (PageAttribute) - (BaseAddress - PageStartAddress);
 978         Attributes        = GetAttributesFromPageEntry (PageEntry);
 979       }
 980
 981       Length = MIN (PageLength, MemorySpaceLength);
 982       if (Attributes != (MemorySpaceMap[Index].Attributes &
 983                          EFI_MEMORY_PAGETYPE_MASK)) {
 984         NewAttributes = (MemorySpaceMap[Index].Attributes &
 985                          ~EFI_MEMORY_PAGETYPE_MASK) | Attributes;
 986         Status = gDS->SetMemorySpaceAttributes (
 987                         BaseAddress,
 988                         Length,
 989                         NewAttributes
 990                         );
 991         ASSERT_EFI_ERROR (Status);
 992         DEBUG ((
 993           DEBUG_VERBOSE,
 994           "Updated memory space attribute: [%lu] %016lx - %016lx (%016lx -> %016lx)\r\n",
 995           (UINT64)Index, BaseAddress, BaseAddress + Length - 1,
 996           MemorySpaceMap[Index].Attributes,
 997           NewAttributes
 998           ));
 999       }
1000
1001       PageLength        -= Length;
1002       MemorySpaceLength -= Length;
1003       BaseAddress       += Length;
1004     }
1005   }
1006
1007   FreePool (MemorySpaceMap);
1008 }
1009
1010 /**
1011   Initialize a buffer pool for page table use only.
1012
1013   To reduce the potential split operation on page table, the pages reserved for
1014   page table should be allocated in the times of PAGE_TABLE_POOL_UNIT_PAGES and
1015   at the boundary of PAGE_TABLE_POOL_ALIGNMENT. So the page pool is always
1016   initialized with number of pages greater than or equal to the given PoolPages.
1017
1018   Once the pages in the pool are used up, this method should be called again to
1019   reserve at least another PAGE_TABLE_POOL_UNIT_PAGES. Usually this won't happen
1020   often in practice.
1021
1022   @param[in] PoolPages      The least page number of the pool to be created.
1023
1024   @retval TRUE    The pool is initialized successfully.
1025   @retval FALSE   The memory is out of resource.
1026 **/
1027 BOOLEAN
1028 InitializePageTablePool (
1029   IN  UINTN                           PoolPages
1030   )
1031 {
1032   VOID                      *Buffer;
1033   BOOLEAN                   IsModified;
1034
1035   //
1036   // Always reserve at least PAGE_TABLE_POOL_UNIT_PAGES, including one page for
1037   // header.
1038   //
1039   PoolPages += 1;   // Add one page for header.
1040   PoolPages = ((PoolPages - 1) / PAGE_TABLE_POOL_UNIT_PAGES + 1) *
1041               PAGE_TABLE_POOL_UNIT_PAGES;
1042   Buffer = AllocateAlignedPages (PoolPages, PAGE_TABLE_POOL_ALIGNMENT);
1043   if (Buffer == NULL) {
1044     DEBUG ((DEBUG_ERROR, "ERROR: Out of aligned pages\r\n"));
1045     return FALSE;
1046   }
1047
1048   //
1049   // Link all pools into a list for easier track later.
1050   //
1051   if (mPageTablePool == NULL) {
1052     mPageTablePool = Buffer;
1053     mPageTablePool->NextPool = mPageTablePool;
1054   } else {
1055     ((PAGE_TABLE_POOL *)Buffer)->NextPool = mPageTablePool->NextPool;
1056     mPageTablePool->NextPool = Buffer;
1057     mPageTablePool = Buffer;
1058   }
1059
1060   //
1061   // Reserve one page for pool header.
1062   //
1063   mPageTablePool->FreePages  = PoolPages - 1;
1064   mPageTablePool->Offset = EFI_PAGES_TO_SIZE (1);
1065
1066   //
1067   // Mark the whole pool pages as read-only.
1068   //
1069   ConvertMemoryPageAttributes (
1070     NULL,
1071     (PHYSICAL_ADDRESS)(UINTN)Buffer,
1072     EFI_PAGES_TO_SIZE (PoolPages),
1073     EFI_MEMORY_RO,
1074     PageActionSet,
1075     AllocatePageTableMemory,
1076     NULL,
1077     &IsModified
1078     );
1079   ASSERT (IsModified == TRUE);
1080
1081   return TRUE;
1082 }
1083
1084 /**
1085   This API provides a way to allocate memory for page table.
1086
1087   This API can be called more than once to allocate memory for page tables.
1088
1089   Allocates the number of 4KB pages and returns a pointer to the allocated
1090   buffer. The buffer returned is aligned on a 4KB boundary.
1091
1092   If Pages is 0, then NULL is returned.
1093   If there is not enough memory remaining to satisfy the request, then NULL is
1094   returned.
1095
1096   @param  Pages                 The number of 4 KB pages to allocate.
1097
1098   @return A pointer to the allocated buffer or NULL if allocation fails.
1099
1100 **/
1101 VOID *
1102 EFIAPI
1103 AllocatePageTableMemory (
1104   IN UINTN           Pages
1105   )
1106 {
1107   VOID                            *Buffer;
1108
1109   if (Pages == 0) {
1110     return NULL;
1111   }
1112
1113   //
1114   // Renew the pool if necessary.
1115   //
1116   if (mPageTablePool == NULL ||
1117       Pages > mPageTablePool->FreePages) {
1118     if (!InitializePageTablePool (Pages)) {
1119       return NULL;
1120     }
1121   }
1122
1123   Buffer = (UINT8 *)mPageTablePool + mPageTablePool->Offset;
1124
1125   mPageTablePool->Offset     += EFI_PAGES_TO_SIZE (Pages);
1126   mPageTablePool->FreePages  -= Pages;
1127
1128   return Buffer;
1129 }
1130
1131 /**
1132   Initialize the Page Table lib.
1133 **/
1134 VOID
1135 InitializePageTableLib (
1136   VOID
1137   )
1138 {
1139   PAGE_TABLE_LIB_PAGING_CONTEXT     CurrentPagingContext;
1140
1141   GetCurrentPagingContext (&CurrentPagingContext);
1142
1143   //
1144   // Reserve memory of page tables for future uses, if paging is enabled.
1145   //
1146   if (CurrentPagingContext.ContextData.X64.PageTableBase != 0 &&
1147       (CurrentPagingContext.ContextData.Ia32.Attributes &
1148        PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE) != 0) {
1149     DisableReadOnlyPageWriteProtect ();
1150     InitializePageTablePool (1);
1151     EnableReadOnlyPageWriteProtect ();
1152   }
1153
1154   DEBUG ((DEBUG_INFO, "CurrentPagingContext:\n", CurrentPagingContext.MachineType));
1155   DEBUG ((DEBUG_INFO, "  MachineType   - 0x%x\n", CurrentPagingContext.MachineType));
1156   DEBUG ((DEBUG_INFO, "  PageTableBase - 0x%x\n", CurrentPagingContext.ContextData.X64.PageTableBase));
1157   DEBUG ((DEBUG_INFO, "  Attributes    - 0x%x\n", CurrentPagingContext.ContextData.X64.Attributes));
1158
1159   return ;
1160 }
1161