UefiCpuPkg/CpuDxe/CpuPageTable.c

   1 /** @file
   2   Page table management support.
   3
   4   Copyright (c) 2017, Intel Corporation. All rights reserved.<BR>
   5   Copyright (c) 2017, AMD Incorporated. All rights reserved.<BR>
   6
   7   This program and the accompanying materials
   8   are licensed and made available under the terms and conditions of the BSD License
   9   which accompanies this distribution.  The full text of the license may be found at
  10   http://opensource.org/licenses/bsd-license.php
  11
  12   THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  13   WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  14
  15 **/
  16
  17 #include <Base.h>
  18 #include <Uefi.h>
  19 #include <Library/BaseLib.h>
  20 #include <Library/CpuLib.h>
  21 #include <Library/BaseMemoryLib.h>
  22 #include <Library/MemoryAllocationLib.h>
  23 #include <Library/DebugLib.h>
  24 #include <Library/UefiBootServicesTableLib.h>
  25 #include <Protocol/MpService.h>
  26
  27 #include "CpuDxe.h"
  28 #include "CpuPageTable.h"
  29
  30 ///
  31 /// Page Table Entry
  32 ///
  33 #define IA32_PG_P                   BIT0
  34 #define IA32_PG_RW                  BIT1
  35 #define IA32_PG_U                   BIT2
  36 #define IA32_PG_WT                  BIT3
  37 #define IA32_PG_CD                  BIT4
  38 #define IA32_PG_A                   BIT5
  39 #define IA32_PG_D                   BIT6
  40 #define IA32_PG_PS                  BIT7
  41 #define IA32_PG_PAT_2M              BIT12
  42 #define IA32_PG_PAT_4K              IA32_PG_PS
  43 #define IA32_PG_PMNT                BIT62
  44 #define IA32_PG_NX                  BIT63
  45
  46 #define PAGE_ATTRIBUTE_BITS         (IA32_PG_D | IA32_PG_A | IA32_PG_U | IA32_PG_RW | IA32_PG_P)
  47 //
  48 // Bits 1, 2, 5, 6 are reserved in the IA32 PAE PDPTE
  49 // X64 PAE PDPTE does not have such restriction
  50 //
  51 #define IA32_PAE_PDPTE_ATTRIBUTE_BITS    (IA32_PG_P)
  52
  53 #define PAGE_PROGATE_BITS           (IA32_PG_NX | PAGE_ATTRIBUTE_BITS)
  54
  55 #define PAGING_4K_MASK  0xFFF
  56 #define PAGING_2M_MASK  0x1FFFFF
  57 #define PAGING_1G_MASK  0x3FFFFFFF
  58
  59 #define PAGING_PAE_INDEX_MASK  0x1FF
  60
  61 #define PAGING_4K_ADDRESS_MASK_64 0x000FFFFFFFFFF000ull
  62 #define PAGING_2M_ADDRESS_MASK_64 0x000FFFFFFFE00000ull
  63 #define PAGING_1G_ADDRESS_MASK_64 0x000FFFFFC0000000ull
  64
  65 typedef enum {
  66   PageNone,
  67   Page4K,
  68   Page2M,
  69   Page1G,
  70 } PAGE_ATTRIBUTE;
  71
  72 typedef struct {
  73   PAGE_ATTRIBUTE   Attribute;
  74   UINT64           Length;
  75   UINT64           AddressMask;
  76 } PAGE_ATTRIBUTE_TABLE;
  77
  78 typedef enum {
  79   PageActionAssign,
  80   PageActionSet,
  81   PageActionClear,
  82 } PAGE_ACTION;
  83
  84 PAGE_ATTRIBUTE_TABLE mPageAttributeTable[] = {
  85   {Page4K,  SIZE_4KB, PAGING_4K_ADDRESS_MASK_64},
  86   {Page2M,  SIZE_2MB, PAGING_2M_ADDRESS_MASK_64},
  87   {Page1G,  SIZE_1GB, PAGING_1G_ADDRESS_MASK_64},
  88 };
  89
  90 PAGE_TABLE_POOL   *mPageTablePool = NULL;
  91
  92 /**
  93   Enable write protection function for AP.
  94
  95   @param[in,out] Buffer  The pointer to private data buffer.
  96 **/
  97 VOID
  98 EFIAPI
  99 SyncCpuEnableWriteProtection (
 100   IN OUT VOID *Buffer
 101   )
 102 {
 103   AsmWriteCr0 (AsmReadCr0 () | BIT16);
 104 }
 105
 106 /**
 107   CpuFlushTlb function for AP.
 108
 109   @param[in,out] Buffer  The pointer to private data buffer.
 110 **/
 111 VOID
 112 EFIAPI
 113 SyncCpuFlushTlb (
 114   IN OUT VOID *Buffer
 115   )
 116 {
 117   CpuFlushTlb();
 118 }
 119
 120 /**
 121   Sync memory page attributes for AP.
 122
 123   @param[in] Procedure            A pointer to the function to be run on enabled APs of
 124                                   the system.
 125 **/
 126 VOID
 127 SyncMemoryPageAttributesAp (
 128   IN EFI_AP_PROCEDURE            Procedure
 129   )
 130 {
 131   EFI_STATUS                Status;
 132   EFI_MP_SERVICES_PROTOCOL  *MpService;
 133
 134   Status = gBS->LocateProtocol (
 135                   &gEfiMpServiceProtocolGuid,
 136                   NULL,
 137                   (VOID **)&MpService
 138                   );
 139   //
 140   // Synchronize the update with all APs
 141   //
 142   if (!EFI_ERROR (Status)) {
 143     Status = MpService->StartupAllAPs (
 144                           MpService,          // This
 145                           Procedure,          // Procedure
 146                           FALSE,              // SingleThread
 147                           NULL,               // WaitEvent
 148                           0,                  // TimeoutInMicrosecsond
 149                           NULL,               // ProcedureArgument
 150                           NULL                // FailedCpuList
 151                           );
 152     ASSERT (Status == EFI_SUCCESS || Status == EFI_NOT_STARTED || Status == EFI_NOT_READY);
 153   }
 154 }
 155
 156 /**
 157   Return current paging context.
 158
 159   @param[in,out]  PagingContext     The paging context.
 160 **/
 161 VOID
 162 GetCurrentPagingContext (
 163   IN OUT PAGE_TABLE_LIB_PAGING_CONTEXT     *PagingContext
 164   )
 165 {
 166   UINT32                         RegEax;
 167   UINT32                         RegEdx;
 168
 169   ZeroMem(PagingContext, sizeof(*PagingContext));
 170   if (sizeof(UINTN) == sizeof(UINT64)) {
 171     PagingContext->MachineType = IMAGE_FILE_MACHINE_X64;
 172   } else {
 173     PagingContext->MachineType = IMAGE_FILE_MACHINE_I386;
 174   }
 175   if ((AsmReadCr0 () & BIT31) != 0) {
 176     PagingContext->ContextData.X64.PageTableBase = (AsmReadCr3 () & PAGING_4K_ADDRESS_MASK_64);
 177   } else {
 178     PagingContext->ContextData.X64.PageTableBase = 0;
 179   }
 180
 181   if ((AsmReadCr4 () & BIT4) != 0) {
 182     PagingContext->ContextData.Ia32.Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PSE;
 183   }
 184   if ((AsmReadCr4 () & BIT5) != 0) {
 185     PagingContext->ContextData.Ia32.Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE;
 186   }
 187   if ((AsmReadCr0 () & BIT16) != 0) {
 188     PagingContext->ContextData.Ia32.Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_WP_ENABLE;
 189   }
 190
 191   AsmCpuid (0x80000000, &RegEax, NULL, NULL, NULL);
 192   if (RegEax > 0x80000000) {
 193     AsmCpuid (0x80000001, NULL, NULL, NULL, &RegEdx);
 194     if ((RegEdx & BIT20) != 0) {
 195       // XD supported
 196       if ((AsmReadMsr64 (0xC0000080) & BIT11) != 0) {
 197         // XD activated
 198         PagingContext->ContextData.Ia32.Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_XD_ACTIVATED;
 199       }
 200     }
 201     if ((RegEdx & BIT26) != 0) {
 202       PagingContext->ContextData.Ia32.Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAGE_1G_SUPPORT;
 203     }
 204   }
 205 }
 206
 207 /**
 208   Return length according to page attributes.
 209
 210   @param[in]  PageAttributes   The page attribute of the page entry.
 211
 212   @return The length of page entry.
 213 **/
 214 UINTN
 215 PageAttributeToLength (
 216   IN PAGE_ATTRIBUTE  PageAttribute
 217   )
 218 {
 219   UINTN  Index;
 220   for (Index = 0; Index < sizeof(mPageAttributeTable)/sizeof(mPageAttributeTable[0]); Index++) {
 221     if (PageAttribute == mPageAttributeTable[Index].Attribute) {
 222       return (UINTN)mPageAttributeTable[Index].Length;
 223     }
 224   }
 225   return 0;
 226 }
 227
 228 /**
 229   Return address mask according to page attributes.
 230
 231   @param[in]  PageAttributes   The page attribute of the page entry.
 232
 233   @return The address mask of page entry.
 234 **/
 235 UINTN
 236 PageAttributeToMask (
 237   IN PAGE_ATTRIBUTE  PageAttribute
 238   )
 239 {
 240   UINTN  Index;
 241   for (Index = 0; Index < sizeof(mPageAttributeTable)/sizeof(mPageAttributeTable[0]); Index++) {
 242     if (PageAttribute == mPageAttributeTable[Index].Attribute) {
 243       return (UINTN)mPageAttributeTable[Index].AddressMask;
 244     }
 245   }
 246   return 0;
 247 }
 248
 249 /**
 250   Return page table entry to match the address.
 251
 252   @param[in]  PagingContext     The paging context.
 253   @param[in]  Address           The address to be checked.
 254   @param[out] PageAttributes    The page attribute of the page entry.
 255
 256   @return The page entry.
 257 **/
 258 VOID *
 259 GetPageTableEntry (
 260   IN  PAGE_TABLE_LIB_PAGING_CONTEXT     *PagingContext,
 261   IN  PHYSICAL_ADDRESS                  Address,
 262   OUT PAGE_ATTRIBUTE                    *PageAttribute
 263   )
 264 {
 265   UINTN                 Index1;
 266   UINTN                 Index2;
 267   UINTN                 Index3;
 268   UINTN                 Index4;
 269   UINT64                *L1PageTable;
 270   UINT64                *L2PageTable;
 271   UINT64                *L3PageTable;
 272   UINT64                *L4PageTable;
 273   UINT64                AddressEncMask;
 274
 275   ASSERT (PagingContext != NULL);
 276
 277   Index4 = ((UINTN)RShiftU64 (Address, 39)) & PAGING_PAE_INDEX_MASK;
 278   Index3 = ((UINTN)Address >> 30) & PAGING_PAE_INDEX_MASK;
 279   Index2 = ((UINTN)Address >> 21) & PAGING_PAE_INDEX_MASK;
 280   Index1 = ((UINTN)Address >> 12) & PAGING_PAE_INDEX_MASK;
 281
 282   // Make sure AddressEncMask is contained to smallest supported address field.
 283   //
 284   AddressEncMask = PcdGet64 (PcdPteMemoryEncryptionAddressOrMask) & PAGING_1G_ADDRESS_MASK_64;
 285
 286   if (PagingContext->MachineType == IMAGE_FILE_MACHINE_X64) {
 287     L4PageTable = (UINT64 *)(UINTN)PagingContext->ContextData.X64.PageTableBase;
 288     if (L4PageTable[Index4] == 0) {
 289       *PageAttribute = PageNone;
 290       return NULL;
 291     }
 292
 293     L3PageTable = (UINT64 *)(UINTN)(L4PageTable[Index4] & ~AddressEncMask & PAGING_4K_ADDRESS_MASK_64);
 294   } else {
 295     ASSERT((PagingContext->ContextData.Ia32.Attributes & PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE) != 0);
 296     L3PageTable = (UINT64 *)(UINTN)PagingContext->ContextData.Ia32.PageTableBase;
 297   }
 298   if (L3PageTable[Index3] == 0) {
 299     *PageAttribute = PageNone;
 300     return NULL;
 301   }
 302   if ((L3PageTable[Index3] & IA32_PG_PS) != 0) {
 303     // 1G
 304     *PageAttribute = Page1G;
 305     return &L3PageTable[Index3];
 306   }
 307
 308   L2PageTable = (UINT64 *)(UINTN)(L3PageTable[Index3] & ~AddressEncMask & PAGING_4K_ADDRESS_MASK_64);
 309   if (L2PageTable[Index2] == 0) {
 310     *PageAttribute = PageNone;
 311     return NULL;
 312   }
 313   if ((L2PageTable[Index2] & IA32_PG_PS) != 0) {
 314     // 2M
 315     *PageAttribute = Page2M;
 316     return &L2PageTable[Index2];
 317   }
 318
 319   // 4k
 320   L1PageTable = (UINT64 *)(UINTN)(L2PageTable[Index2] & ~AddressEncMask & PAGING_4K_ADDRESS_MASK_64);
 321   if ((L1PageTable[Index1] == 0) && (Address != 0)) {
 322     *PageAttribute = PageNone;
 323     return NULL;
 324   }
 325   *PageAttribute = Page4K;
 326   return &L1PageTable[Index1];
 327 }
 328
 329 /**
 330   Return memory attributes of page entry.
 331
 332   @param[in]  PageEntry        The page entry.
 333
 334   @return Memory attributes of page entry.
 335 **/
 336 UINT64
 337 GetAttributesFromPageEntry (
 338   IN  UINT64                            *PageEntry
 339   )
 340 {
 341   UINT64  Attributes;
 342   Attributes = 0;
 343   if ((*PageEntry & IA32_PG_P) == 0) {
 344     Attributes |= EFI_MEMORY_RP;
 345   }
 346   if ((*PageEntry & IA32_PG_RW) == 0) {
 347     Attributes |= EFI_MEMORY_RO;
 348   }
 349   if ((*PageEntry & IA32_PG_NX) != 0) {
 350     Attributes |= EFI_MEMORY_XP;
 351   }
 352   return Attributes;
 353 }
 354
 355 /**
 356   Modify memory attributes of page entry.
 357
 358   @param[in]  PagingContext    The paging context.
 359   @param[in]  PageEntry        The page entry.
 360   @param[in]  Attributes       The bit mask of attributes to modify for the memory region.
 361   @param[in]  PageAction       The page action.
 362   @param[out] IsModified       TRUE means page table modified. FALSE means page table not modified.
 363 **/
 364 VOID
 365 ConvertPageEntryAttribute (
 366   IN  PAGE_TABLE_LIB_PAGING_CONTEXT     *PagingContext,
 367   IN  UINT64                            *PageEntry,
 368   IN  UINT64                            Attributes,
 369   IN  PAGE_ACTION                       PageAction,
 370   OUT BOOLEAN                           *IsModified
 371   )
 372 {
 373   UINT64  CurrentPageEntry;
 374   UINT64  NewPageEntry;
 375
 376   CurrentPageEntry = *PageEntry;
 377   NewPageEntry = CurrentPageEntry;
 378   if ((Attributes & EFI_MEMORY_RP) != 0) {
 379     switch (PageAction) {
 380     case PageActionAssign:
 381     case PageActionSet:
 382       NewPageEntry &= ~(UINT64)IA32_PG_P;
 383       break;
 384     case PageActionClear:
 385       NewPageEntry |= IA32_PG_P;
 386       break;
 387     }
 388   } else {
 389     switch (PageAction) {
 390     case PageActionAssign:
 391       NewPageEntry |= IA32_PG_P;
 392       break;
 393     case PageActionSet:
 394     case PageActionClear:
 395       break;
 396     }
 397   }
 398   if ((Attributes & EFI_MEMORY_RO) != 0) {
 399     switch (PageAction) {
 400     case PageActionAssign:
 401     case PageActionSet:
 402       NewPageEntry &= ~(UINT64)IA32_PG_RW;
 403       break;
 404     case PageActionClear:
 405       NewPageEntry |= IA32_PG_RW;
 406       break;
 407     }
 408   } else {
 409     switch (PageAction) {
 410     case PageActionAssign:
 411       NewPageEntry |= IA32_PG_RW;
 412       break;
 413     case PageActionSet:
 414     case PageActionClear:
 415       break;
 416     }
 417   }
 418   if ((PagingContext->ContextData.Ia32.Attributes & PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_XD_ACTIVATED) != 0) {
 419     if ((Attributes & EFI_MEMORY_XP) != 0) {
 420       switch (PageAction) {
 421       case PageActionAssign:
 422       case PageActionSet:
 423         NewPageEntry |= IA32_PG_NX;
 424         break;
 425       case PageActionClear:
 426         NewPageEntry &= ~IA32_PG_NX;
 427         break;
 428       }
 429     } else {
 430       switch (PageAction) {
 431       case PageActionAssign:
 432         NewPageEntry &= ~IA32_PG_NX;
 433         break;
 434       case PageActionSet:
 435       case PageActionClear:
 436         break;
 437       }
 438     }
 439   }
 440   *PageEntry = NewPageEntry;
 441   if (CurrentPageEntry != NewPageEntry) {
 442     *IsModified = TRUE;
 443     DEBUG ((DEBUG_VERBOSE, "ConvertPageEntryAttribute 0x%lx", CurrentPageEntry));
 444     DEBUG ((DEBUG_VERBOSE, "->0x%lx\n", NewPageEntry));
 445   } else {
 446     *IsModified = FALSE;
 447   }
 448 }
 449
 450 /**
 451   This function returns if there is need to split page entry.
 452
 453   @param[in]  BaseAddress      The base address to be checked.
 454   @param[in]  Length           The length to be checked.
 455   @param[in]  PageEntry        The page entry to be checked.
 456   @param[in]  PageAttribute    The page attribute of the page entry.
 457
 458   @retval SplitAttributes on if there is need to split page entry.
 459 **/
 460 PAGE_ATTRIBUTE
 461 NeedSplitPage (
 462   IN  PHYSICAL_ADDRESS                  BaseAddress,
 463   IN  UINT64                            Length,
 464   IN  UINT64                            *PageEntry,
 465   IN  PAGE_ATTRIBUTE                    PageAttribute
 466   )
 467 {
 468   UINT64                PageEntryLength;
 469
 470   PageEntryLength = PageAttributeToLength (PageAttribute);
 471
 472   if (((BaseAddress & (PageEntryLength - 1)) == 0) && (Length >= PageEntryLength)) {
 473     return PageNone;
 474   }
 475
 476   if (((BaseAddress & PAGING_2M_MASK) != 0) || (Length < SIZE_2MB)) {
 477     return Page4K;
 478   }
 479
 480   return Page2M;
 481 }
 482
 483 /**
 484   This function splits one page entry to small page entries.
 485
 486   @param[in]  PageEntry         The page entry to be splitted.
 487   @param[in]  PageAttribute     The page attribute of the page entry.
 488   @param[in]  SplitAttribute    How to split the page entry.
 489   @param[in]  AllocatePagesFunc If page split is needed, this function is used to allocate more pages.
 490
 491   @retval RETURN_SUCCESS            The page entry is splitted.
 492   @retval RETURN_UNSUPPORTED        The page entry does not support to be splitted.
 493   @retval RETURN_OUT_OF_RESOURCES   No resource to split page entry.
 494 **/
 495 RETURN_STATUS
 496 SplitPage (
 497   IN  UINT64                            *PageEntry,
 498   IN  PAGE_ATTRIBUTE                    PageAttribute,
 499   IN  PAGE_ATTRIBUTE                    SplitAttribute,
 500   IN  PAGE_TABLE_LIB_ALLOCATE_PAGES     AllocatePagesFunc
 501   )
 502 {
 503   UINT64   BaseAddress;
 504   UINT64   *NewPageEntry;
 505   UINTN    Index;
 506   UINT64   AddressEncMask;
 507
 508   ASSERT (PageAttribute == Page2M || PageAttribute == Page1G);
 509
 510   ASSERT (AllocatePagesFunc != NULL);
 511
 512   // Make sure AddressEncMask is contained to smallest supported address field.
 513   //
 514   AddressEncMask = PcdGet64 (PcdPteMemoryEncryptionAddressOrMask) & PAGING_1G_ADDRESS_MASK_64;
 515
 516   if (PageAttribute == Page2M) {
 517     //
 518     // Split 2M to 4K
 519     //
 520     ASSERT (SplitAttribute == Page4K);
 521     if (SplitAttribute == Page4K) {
 522       NewPageEntry = AllocatePagesFunc (1);
 523       DEBUG ((DEBUG_INFO, "Split - 0x%x\n", NewPageEntry));
 524       if (NewPageEntry == NULL) {
 525         return RETURN_OUT_OF_RESOURCES;
 526       }
 527       BaseAddress = *PageEntry & ~AddressEncMask & PAGING_2M_ADDRESS_MASK_64;
 528       for (Index = 0; Index < SIZE_4KB / sizeof(UINT64); Index++) {
 529         NewPageEntry[Index] = (BaseAddress + SIZE_4KB * Index) | AddressEncMask | ((*PageEntry) & PAGE_PROGATE_BITS);
 530       }
 531       (*PageEntry) = (UINT64)(UINTN)NewPageEntry | AddressEncMask | ((*PageEntry) & PAGE_ATTRIBUTE_BITS);
 532       return RETURN_SUCCESS;
 533     } else {
 534       return RETURN_UNSUPPORTED;
 535     }
 536   } else if (PageAttribute == Page1G) {
 537     //
 538     // Split 1G to 2M
 539     // No need support 1G->4K directly, we should use 1G->2M, then 2M->4K to get more compact page table.
 540     //
 541     ASSERT (SplitAttribute == Page2M || SplitAttribute == Page4K);
 542     if ((SplitAttribute == Page2M || SplitAttribute == Page4K)) {
 543       NewPageEntry = AllocatePagesFunc (1);
 544       DEBUG ((DEBUG_INFO, "Split - 0x%x\n", NewPageEntry));
 545       if (NewPageEntry == NULL) {
 546         return RETURN_OUT_OF_RESOURCES;
 547       }
 548       BaseAddress = *PageEntry & ~AddressEncMask  & PAGING_1G_ADDRESS_MASK_64;
 549       for (Index = 0; Index < SIZE_4KB / sizeof(UINT64); Index++) {
 550         NewPageEntry[Index] = (BaseAddress + SIZE_2MB * Index) | AddressEncMask | IA32_PG_PS | ((*PageEntry) & PAGE_PROGATE_BITS);
 551       }
 552       (*PageEntry) = (UINT64)(UINTN)NewPageEntry | AddressEncMask | ((*PageEntry) & PAGE_ATTRIBUTE_BITS);
 553       return RETURN_SUCCESS;
 554     } else {
 555       return RETURN_UNSUPPORTED;
 556     }
 557   } else {
 558     return RETURN_UNSUPPORTED;
 559   }
 560 }
 561
 562 /**
 563  Check the WP status in CR0 register. This bit is used to lock or unlock write
 564  access to pages marked as read-only.
 565
 566   @retval TRUE    Write protection is enabled.
 567   @retval FALSE   Write protection is disabled.
 568 **/
 569 BOOLEAN
 570 IsReadOnlyPageWriteProtected (
 571   VOID
 572   )
 573 {
 574   return ((AsmReadCr0 () & BIT16) != 0);
 575 }
 576
 577 /**
 578   Disable write protection function for AP.
 579
 580   @param[in,out] Buffer  The pointer to private data buffer.
 581 **/
 582 VOID
 583 EFIAPI
 584 SyncCpuDisableWriteProtection (
 585   IN OUT VOID *Buffer
 586   )
 587 {
 588   AsmWriteCr0 (AsmReadCr0() & ~BIT16);
 589 }
 590
 591 /**
 592  Disable Write Protect on pages marked as read-only.
 593 **/
 594 VOID
 595 DisableReadOnlyPageWriteProtect (
 596   VOID
 597   )
 598 {
 599   AsmWriteCr0 (AsmReadCr0() & ~BIT16);
 600 }
 601
 602 /**
 603  Enable Write Protect on pages marked as read-only.
 604 **/
 605 VOID
 606 EnableReadOnlyPageWriteProtect (
 607   VOID
 608   )
 609 {
 610   AsmWriteCr0 (AsmReadCr0() | BIT16);
 611 }
 612
 613 /**
 614   This function modifies the page attributes for the memory region specified by BaseAddress and
 615   Length from their current attributes to the attributes specified by Attributes.
 616
 617   Caller should make sure BaseAddress and Length is at page boundary.
 618
 619   @param[in]  PagingContext     The paging context. NULL means get page table from current CPU context.
 620   @param[in]  BaseAddress       The physical address that is the start address of a memory region.
 621   @param[in]  Length            The size in bytes of the memory region.
 622   @param[in]  Attributes        The bit mask of attributes to modify for the memory region.
 623   @param[in]  PageAction        The page action.
 624   @param[in]  AllocatePagesFunc If page split is needed, this function is used to allocate more pages.
 625                                 NULL mean page split is unsupported.
 626   @param[out] IsSplitted        TRUE means page table splitted. FALSE means page table not splitted.
 627   @param[out] IsModified        TRUE means page table modified. FALSE means page table not modified.
 628
 629   @retval RETURN_SUCCESS           The attributes were modified for the memory region.
 630   @retval RETURN_ACCESS_DENIED     The attributes for the memory resource range specified by
 631                                    BaseAddress and Length cannot be modified.
 632   @retval RETURN_INVALID_PARAMETER Length is zero.
 633                                    Attributes specified an illegal combination of attributes that
 634                                    cannot be set together.
 635   @retval RETURN_OUT_OF_RESOURCES  There are not enough system resources to modify the attributes of
 636                                    the memory resource range.
 637   @retval RETURN_UNSUPPORTED       The processor does not support one or more bytes of the memory
 638                                    resource range specified by BaseAddress and Length.
 639                                    The bit mask of attributes is not support for the memory resource
 640                                    range specified by BaseAddress and Length.
 641 **/
 642 RETURN_STATUS
 643 ConvertMemoryPageAttributes (
 644   IN  PAGE_TABLE_LIB_PAGING_CONTEXT     *PagingContext OPTIONAL,
 645   IN  PHYSICAL_ADDRESS                  BaseAddress,
 646   IN  UINT64                            Length,
 647   IN  UINT64                            Attributes,
 648   IN  PAGE_ACTION                       PageAction,
 649   IN  PAGE_TABLE_LIB_ALLOCATE_PAGES     AllocatePagesFunc OPTIONAL,
 650   OUT BOOLEAN                           *IsSplitted,  OPTIONAL
 651   OUT BOOLEAN                           *IsModified   OPTIONAL
 652   )
 653 {
 654   PAGE_TABLE_LIB_PAGING_CONTEXT     CurrentPagingContext;
 655   UINT64                            *PageEntry;
 656   PAGE_ATTRIBUTE                    PageAttribute;
 657   UINTN                             PageEntryLength;
 658   PAGE_ATTRIBUTE                    SplitAttribute;
 659   RETURN_STATUS                     Status;
 660   BOOLEAN                           IsEntryModified;
 661   BOOLEAN                           IsWpEnabled;
 662
 663   if ((BaseAddress & (SIZE_4KB - 1)) != 0) {
 664     DEBUG ((DEBUG_ERROR, "BaseAddress(0x%lx) is not aligned!\n", BaseAddress));
 665     return EFI_UNSUPPORTED;
 666   }
 667   if ((Length & (SIZE_4KB - 1)) != 0) {
 668     DEBUG ((DEBUG_ERROR, "Length(0x%lx) is not aligned!\n", Length));
 669     return EFI_UNSUPPORTED;
 670   }
 671   if (Length == 0) {
 672     DEBUG ((DEBUG_ERROR, "Length is 0!\n"));
 673     return RETURN_INVALID_PARAMETER;
 674   }
 675
 676   if ((Attributes & ~(EFI_MEMORY_RP | EFI_MEMORY_RO | EFI_MEMORY_XP)) != 0) {
 677     DEBUG ((DEBUG_ERROR, "Attributes(0x%lx) has unsupported bit\n", Attributes));
 678     return EFI_UNSUPPORTED;
 679   }
 680
 681   if (PagingContext == NULL) {
 682     GetCurrentPagingContext (&CurrentPagingContext);
 683   } else {
 684     CopyMem (&CurrentPagingContext, PagingContext, sizeof(CurrentPagingContext));
 685   }
 686   switch(CurrentPagingContext.MachineType) {
 687   case IMAGE_FILE_MACHINE_I386:
 688     if (CurrentPagingContext.ContextData.Ia32.PageTableBase == 0) {
 689       if (Attributes == 0) {
 690         return EFI_SUCCESS;
 691       } else {
 692         DEBUG ((DEBUG_ERROR, "PageTable is 0!\n"));
 693         return EFI_UNSUPPORTED;
 694       }
 695     }
 696     if ((CurrentPagingContext.ContextData.Ia32.Attributes & PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE) == 0) {
 697       DEBUG ((DEBUG_ERROR, "Non-PAE Paging!\n"));
 698       return EFI_UNSUPPORTED;
 699     }
 700     if ((BaseAddress + Length) > BASE_4GB) {
 701       DEBUG ((DEBUG_ERROR, "Beyond 4GB memory in 32-bit mode!\n"));
 702       return EFI_UNSUPPORTED;
 703     }
 704     break;
 705   case IMAGE_FILE_MACHINE_X64:
 706     ASSERT (CurrentPagingContext.ContextData.X64.PageTableBase != 0);
 707     break;
 708   default:
 709     ASSERT(FALSE);
 710     return EFI_UNSUPPORTED;
 711     break;
 712   }
 713
 714 //  DEBUG ((DEBUG_ERROR, "ConvertMemoryPageAttributes(%x) - %016lx, %016lx, %02lx\n", IsSet, BaseAddress, Length, Attributes));
 715
 716   if (IsSplitted != NULL) {
 717     *IsSplitted = FALSE;
 718   }
 719   if (IsModified != NULL) {
 720     *IsModified = FALSE;
 721   }
 722   if (AllocatePagesFunc == NULL) {
 723     AllocatePagesFunc = AllocatePageTableMemory;
 724   }
 725
 726   //
 727   // Make sure that the page table is changeable.
 728   //
 729   IsWpEnabled = IsReadOnlyPageWriteProtected ();
 730   if (IsWpEnabled) {
 731     DisableReadOnlyPageWriteProtect ();
 732   }
 733
 734   //
 735   // Below logic is to check 2M/4K page to make sure we donot waist memory.
 736   //
 737   Status = EFI_SUCCESS;
 738   while (Length != 0) {
 739     PageEntry = GetPageTableEntry (&CurrentPagingContext, BaseAddress, &PageAttribute);
 740     if (PageEntry == NULL) {
 741       Status = RETURN_UNSUPPORTED;
 742       goto Done;
 743     }
 744     PageEntryLength = PageAttributeToLength (PageAttribute);
 745     SplitAttribute = NeedSplitPage (BaseAddress, Length, PageEntry, PageAttribute);
 746     if (SplitAttribute == PageNone) {
 747       ConvertPageEntryAttribute (&CurrentPagingContext, PageEntry, Attributes, PageAction, &IsEntryModified);
 748       if (IsEntryModified) {
 749         if (IsModified != NULL) {
 750           *IsModified = TRUE;
 751         }
 752       }
 753       //
 754       // Convert success, move to next
 755       //
 756       BaseAddress += PageEntryLength;
 757       Length -= PageEntryLength;
 758     } else {
 759       if (AllocatePagesFunc == NULL) {
 760         Status = RETURN_UNSUPPORTED;
 761         goto Done;
 762       }
 763       Status = SplitPage (PageEntry, PageAttribute, SplitAttribute, AllocatePagesFunc);
 764       if (RETURN_ERROR (Status)) {
 765         Status = RETURN_UNSUPPORTED;
 766         goto Done;
 767       }
 768       if (IsSplitted != NULL) {
 769         *IsSplitted = TRUE;
 770       }
 771       if (IsModified != NULL) {
 772         *IsModified = TRUE;
 773       }
 774       //
 775       // Just split current page
 776       // Convert success in next around
 777       //
 778     }
 779   }
 780
 781 Done:
 782   //
 783   // Restore page table write protection, if any.
 784   //
 785   if (IsWpEnabled) {
 786     EnableReadOnlyPageWriteProtect ();
 787   }
 788   return Status;
 789 }
 790
 791 /**
 792   This function assigns the page attributes for the memory region specified by BaseAddress and
 793   Length from their current attributes to the attributes specified by Attributes.
 794
 795   Caller should make sure BaseAddress and Length is at page boundary.
 796
 797   Caller need guarentee the TPL <= TPL_NOTIFY, if there is split page request.
 798
 799   @param[in]  PagingContext     The paging context. NULL means get page table from current CPU context.
 800   @param[in]  BaseAddress       The physical address that is the start address of a memory region.
 801   @param[in]  Length            The size in bytes of the memory region.
 802   @param[in]  Attributes        The bit mask of attributes to set for the memory region.
 803   @param[in]  AllocatePagesFunc If page split is needed, this function is used to allocate more pages.
 804                                 NULL mean page split is unsupported.
 805
 806   @retval RETURN_SUCCESS           The attributes were cleared for the memory region.
 807   @retval RETURN_ACCESS_DENIED     The attributes for the memory resource range specified by
 808                                    BaseAddress and Length cannot be modified.
 809   @retval RETURN_INVALID_PARAMETER Length is zero.
 810                                    Attributes specified an illegal combination of attributes that
 811                                    cannot be set together.
 812   @retval RETURN_OUT_OF_RESOURCES  There are not enough system resources to modify the attributes of
 813                                    the memory resource range.
 814   @retval RETURN_UNSUPPORTED       The processor does not support one or more bytes of the memory
 815                                    resource range specified by BaseAddress and Length.
 816                                    The bit mask of attributes is not support for the memory resource
 817                                    range specified by BaseAddress and Length.
 818 **/
 819 RETURN_STATUS
 820 EFIAPI
 821 AssignMemoryPageAttributes (
 822   IN  PAGE_TABLE_LIB_PAGING_CONTEXT     *PagingContext OPTIONAL,
 823   IN  PHYSICAL_ADDRESS                  BaseAddress,
 824   IN  UINT64                            Length,
 825   IN  UINT64                            Attributes,
 826   IN  PAGE_TABLE_LIB_ALLOCATE_PAGES     AllocatePagesFunc OPTIONAL
 827   )
 828 {
 829   RETURN_STATUS  Status;
 830   BOOLEAN        IsModified;
 831   BOOLEAN        IsSplitted;
 832
 833 //  DEBUG((DEBUG_INFO, "AssignMemoryPageAttributes: 0x%lx - 0x%lx (0x%lx)\n", BaseAddress, Length, Attributes));
 834   Status = ConvertMemoryPageAttributes (PagingContext, BaseAddress, Length, Attributes, PageActionAssign, AllocatePagesFunc, &IsSplitted, &IsModified);
 835   if (!EFI_ERROR(Status)) {
 836     if ((PagingContext == NULL) && IsModified) {
 837       //
 838       // Flush TLB as last step
 839       //
 840       CpuFlushTlb();
 841       SyncMemoryPageAttributesAp (SyncCpuFlushTlb);
 842     }
 843   }
 844
 845   return Status;
 846 }
 847
 848 /**
 849  Check if Execute Disable feature is enabled or not.
 850 **/
 851 BOOLEAN
 852 IsExecuteDisableEnabled (
 853   VOID
 854   )
 855 {
 856   MSR_CORE_IA32_EFER_REGISTER    MsrEfer;
 857
 858   MsrEfer.Uint64 = AsmReadMsr64 (MSR_IA32_EFER);
 859   return (MsrEfer.Bits.NXE == 1);
 860 }
 861
 862 /**
 863   Update GCD memory space attributes according to current page table setup.
 864 **/
 865 VOID
 866 RefreshGcdMemoryAttributesFromPaging (
 867   VOID
 868   )
 869 {
 870   EFI_STATUS                          Status;
 871   UINTN                               NumberOfDescriptors;
 872   EFI_GCD_MEMORY_SPACE_DESCRIPTOR     *MemorySpaceMap;
 873   PAGE_TABLE_LIB_PAGING_CONTEXT       PagingContext;
 874   PAGE_ATTRIBUTE                      PageAttribute;
 875   UINT64                              *PageEntry;
 876   UINT64                              PageLength;
 877   UINT64                              MemorySpaceLength;
 878   UINT64                              Length;
 879   UINT64                              BaseAddress;
 880   UINT64                              PageStartAddress;
 881   UINT64                              Attributes;
 882   UINT64                              Capabilities;
 883   UINT64                              NewAttributes;
 884   UINTN                               Index;
 885
 886   //
 887   // Assuming that memory space map returned is sorted already; otherwise sort
 888   // them in the order of lowest address to highest address.
 889   //
 890   Status = gDS->GetMemorySpaceMap (&NumberOfDescriptors, &MemorySpaceMap);
 891   ASSERT_EFI_ERROR (Status);
 892
 893   GetCurrentPagingContext (&PagingContext);
 894
 895   Attributes      = 0;
 896   NewAttributes   = 0;
 897   BaseAddress     = 0;
 898   PageLength      = 0;
 899
 900   if (IsExecuteDisableEnabled ()) {
 901     Capabilities = EFI_MEMORY_RO | EFI_MEMORY_RP | EFI_MEMORY_XP;
 902   } else {
 903     Capabilities = EFI_MEMORY_RO | EFI_MEMORY_RP;
 904   }
 905
 906   for (Index = 0; Index < NumberOfDescriptors; Index++) {
 907     if (MemorySpaceMap[Index].GcdMemoryType == EfiGcdMemoryTypeNonExistent) {
 908       continue;
 909     }
 910
 911     //
 912     // Sync the actual paging related capabilities back to GCD service first.
 913     // As a side effect (good one), this can also help to avoid unnecessary
 914     // memory map entries due to the different capabilities of the same type
 915     // memory, such as multiple RT_CODE and RT_DATA entries in memory map,
 916     // which could cause boot failure of some old Linux distro (before v4.3).
 917     //
 918     Status = gDS->SetMemorySpaceCapabilities (
 919                     MemorySpaceMap[Index].BaseAddress,
 920                     MemorySpaceMap[Index].Length,
 921                     MemorySpaceMap[Index].Capabilities | Capabilities
 922                     );
 923     if (EFI_ERROR (Status)) {
 924       //
 925       // If we cannot udpate the capabilities, we cannot update its
 926       // attributes either. So just simply skip current block of memory.
 927       //
 928       DEBUG ((
 929         DEBUG_WARN,
 930         "Failed to update capability: [%lu] %016lx - %016lx (%016lx -> %016lx)\r\n",
 931         (UINT64)Index, MemorySpaceMap[Index].BaseAddress,
 932         MemorySpaceMap[Index].BaseAddress + MemorySpaceMap[Index].Length - 1,
 933         MemorySpaceMap[Index].Capabilities,
 934         MemorySpaceMap[Index].Capabilities | Capabilities
 935         ));
 936       continue;
 937     }
 938
 939     if (MemorySpaceMap[Index].BaseAddress >= (BaseAddress + PageLength)) {
 940       //
 941       // Current memory space starts at a new page. Resetting PageLength will
 942       // trigger a retrieval of page attributes at new address.
 943       //
 944       PageLength = 0;
 945     } else {
 946       //
 947       // In case current memory space is not adjacent to last one
 948       //
 949       PageLength -= (MemorySpaceMap[Index].BaseAddress - BaseAddress);
 950     }
 951
 952     //
 953     // Sync actual page attributes to GCD
 954     //
 955     BaseAddress       = MemorySpaceMap[Index].BaseAddress;
 956     MemorySpaceLength = MemorySpaceMap[Index].Length;
 957     while (MemorySpaceLength > 0) {
 958       if (PageLength == 0) {
 959         PageEntry = GetPageTableEntry (&PagingContext, BaseAddress, &PageAttribute);
 960         if (PageEntry == NULL) {
 961           break;
 962         }
 963
 964         //
 965         // Note current memory space might start in the middle of a page
 966         //
 967         PageStartAddress  = (*PageEntry) & (UINT64)PageAttributeToMask(PageAttribute);
 968         PageLength        = PageAttributeToLength (PageAttribute) - (BaseAddress - PageStartAddress);
 969         Attributes        = GetAttributesFromPageEntry (PageEntry);
 970       }
 971
 972       Length = MIN (PageLength, MemorySpaceLength);
 973       if (Attributes != (MemorySpaceMap[Index].Attributes &
 974                          EFI_MEMORY_PAGETYPE_MASK)) {
 975         NewAttributes = (MemorySpaceMap[Index].Attributes &
 976                          ~EFI_MEMORY_PAGETYPE_MASK) | Attributes;
 977         Status = gDS->SetMemorySpaceAttributes (
 978                         BaseAddress,
 979                         Length,
 980                         NewAttributes
 981                         );
 982         ASSERT_EFI_ERROR (Status);
 983         DEBUG ((
 984           DEBUG_VERBOSE,
 985           "Updated memory space attribute: [%lu] %016lx - %016lx (%016lx -> %016lx)\r\n",
 986           (UINT64)Index, BaseAddress, BaseAddress + Length - 1,
 987           MemorySpaceMap[Index].Attributes,
 988           NewAttributes
 989           ));
 990       }
 991
 992       PageLength        -= Length;
 993       MemorySpaceLength -= Length;
 994       BaseAddress       += Length;
 995     }
 996   }
 997
 998   FreePool (MemorySpaceMap);
 999 }
1000
1001 /**
1002   Initialize a buffer pool for page table use only.
1003
1004   To reduce the potential split operation on page table, the pages reserved for
1005   page table should be allocated in the times of PAGE_TABLE_POOL_UNIT_PAGES and
1006   at the boundary of PAGE_TABLE_POOL_ALIGNMENT. So the page pool is always
1007   initialized with number of pages greater than or equal to the given PoolPages.
1008
1009   Once the pages in the pool are used up, this method should be called again to
1010   reserve at least another PAGE_TABLE_POOL_UNIT_PAGES. Usually this won't happen
1011   often in practice.
1012
1013   @param[in] PoolPages      The least page number of the pool to be created.
1014
1015   @retval TRUE    The pool is initialized successfully.
1016   @retval FALSE   The memory is out of resource.
1017 **/
1018 BOOLEAN
1019 InitializePageTablePool (
1020   IN  UINTN                           PoolPages
1021   )
1022 {
1023   VOID                      *Buffer;
1024   BOOLEAN                   IsModified;
1025
1026   //
1027   // Always reserve at least PAGE_TABLE_POOL_UNIT_PAGES, including one page for
1028   // header.
1029   //
1030   PoolPages += 1;   // Add one page for header.
1031   PoolPages = ((PoolPages - 1) / PAGE_TABLE_POOL_UNIT_PAGES + 1) *
1032               PAGE_TABLE_POOL_UNIT_PAGES;
1033   Buffer = AllocateAlignedPages (PoolPages, PAGE_TABLE_POOL_ALIGNMENT);
1034   if (Buffer == NULL) {
1035     DEBUG ((DEBUG_ERROR, "ERROR: Out of aligned pages\r\n"));
1036     return FALSE;
1037   }
1038
1039   //
1040   // Link all pools into a list for easier track later.
1041   //
1042   if (mPageTablePool == NULL) {
1043     mPageTablePool = Buffer;
1044     mPageTablePool->NextPool = mPageTablePool;
1045   } else {
1046     ((PAGE_TABLE_POOL *)Buffer)->NextPool = mPageTablePool->NextPool;
1047     mPageTablePool->NextPool = Buffer;
1048     mPageTablePool = Buffer;
1049   }
1050
1051   //
1052   // Reserve one page for pool header.
1053   //
1054   mPageTablePool->FreePages  = PoolPages - 1;
1055   mPageTablePool->Offset = EFI_PAGES_TO_SIZE (1);
1056
1057   //
1058   // Mark the whole pool pages as read-only.
1059   //
1060   ConvertMemoryPageAttributes (
1061     NULL,
1062     (PHYSICAL_ADDRESS)(UINTN)Buffer,
1063     EFI_PAGES_TO_SIZE (PoolPages),
1064     EFI_MEMORY_RO,
1065     PageActionSet,
1066     AllocatePageTableMemory,
1067     NULL,
1068     &IsModified
1069     );
1070   ASSERT (IsModified == TRUE);
1071
1072   return TRUE;
1073 }
1074
1075 /**
1076   This API provides a way to allocate memory for page table.
1077
1078   This API can be called more than once to allocate memory for page tables.
1079
1080   Allocates the number of 4KB pages and returns a pointer to the allocated
1081   buffer. The buffer returned is aligned on a 4KB boundary.
1082
1083   If Pages is 0, then NULL is returned.
1084   If there is not enough memory remaining to satisfy the request, then NULL is
1085   returned.
1086
1087   @param  Pages                 The number of 4 KB pages to allocate.
1088
1089   @return A pointer to the allocated buffer or NULL if allocation fails.
1090
1091 **/
1092 VOID *
1093 EFIAPI
1094 AllocatePageTableMemory (
1095   IN UINTN           Pages
1096   )
1097 {
1098   VOID                            *Buffer;
1099
1100   if (Pages == 0) {
1101     return NULL;
1102   }
1103
1104   //
1105   // Renew the pool if necessary.
1106   //
1107   if (mPageTablePool == NULL ||
1108       Pages > mPageTablePool->FreePages) {
1109     if (!InitializePageTablePool (Pages)) {
1110       return NULL;
1111     }
1112   }
1113
1114   Buffer = (UINT8 *)mPageTablePool + mPageTablePool->Offset;
1115
1116   mPageTablePool->Offset     += EFI_PAGES_TO_SIZE (Pages);
1117   mPageTablePool->FreePages  -= Pages;
1118
1119   return Buffer;
1120 }
1121
1122 /**
1123   Initialize the Page Table lib.
1124 **/
1125 VOID
1126 InitializePageTableLib (
1127   VOID
1128   )
1129 {
1130   PAGE_TABLE_LIB_PAGING_CONTEXT     CurrentPagingContext;
1131
1132   GetCurrentPagingContext (&CurrentPagingContext);
1133
1134   //
1135   // Reserve memory of page tables for future uses, if paging is enabled.
1136   //
1137   if (CurrentPagingContext.ContextData.X64.PageTableBase != 0 &&
1138       (CurrentPagingContext.ContextData.Ia32.Attributes &
1139        PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE) != 0) {
1140     DisableReadOnlyPageWriteProtect ();
1141     InitializePageTablePool (1);
1142     EnableReadOnlyPageWriteProtect ();
1143   }
1144
1145   DEBUG ((DEBUG_INFO, "CurrentPagingContext:\n", CurrentPagingContext.MachineType));
1146   DEBUG ((DEBUG_INFO, "  MachineType   - 0x%x\n", CurrentPagingContext.MachineType));
1147   DEBUG ((DEBUG_INFO, "  PageTableBase - 0x%x\n", CurrentPagingContext.ContextData.X64.PageTableBase));
1148   DEBUG ((DEBUG_INFO, "  Attributes    - 0x%x\n", CurrentPagingContext.ContextData.X64.Attributes));
1149
1150   return ;
1151 }
1152