UefiCpuPkg/CpuDxe/CpuPageTable.c

   1 /** @file
   2   Page table management support.
   3
   4   Copyright (c) 2017, Intel Corporation. All rights reserved.<BR>
   5   Copyright (c) 2017, AMD Incorporated. All rights reserved.<BR>
   6
   7   This program and the accompanying materials
   8   are licensed and made available under the terms and conditions of the BSD License
   9   which accompanies this distribution.  The full text of the license may be found at
  10   http://opensource.org/licenses/bsd-license.php
  11
  12   THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  13   WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  14
  15 **/
  16
  17 #include <Base.h>
  18 #include <Uefi.h>
  19 #include <Library/BaseLib.h>
  20 #include <Library/CpuLib.h>
  21 #include <Library/BaseMemoryLib.h>
  22 #include <Library/MemoryAllocationLib.h>
  23 #include <Library/DebugLib.h>
  24 #include <Library/UefiBootServicesTableLib.h>
  25 #include <Protocol/MpService.h>
  26
  27 #include "CpuDxe.h"
  28 #include "CpuPageTable.h"
  29
  30 ///
  31 /// Page Table Entry
  32 ///
  33 #define IA32_PG_P                   BIT0
  34 #define IA32_PG_RW                  BIT1
  35 #define IA32_PG_U                   BIT2
  36 #define IA32_PG_WT                  BIT3
  37 #define IA32_PG_CD                  BIT4
  38 #define IA32_PG_A                   BIT5
  39 #define IA32_PG_D                   BIT6
  40 #define IA32_PG_PS                  BIT7
  41 #define IA32_PG_PAT_2M              BIT12
  42 #define IA32_PG_PAT_4K              IA32_PG_PS
  43 #define IA32_PG_PMNT                BIT62
  44 #define IA32_PG_NX                  BIT63
  45
  46 #define PAGE_ATTRIBUTE_BITS         (IA32_PG_D | IA32_PG_A | IA32_PG_U | IA32_PG_RW | IA32_PG_P)
  47 //
  48 // Bits 1, 2, 5, 6 are reserved in the IA32 PAE PDPTE
  49 // X64 PAE PDPTE does not have such restriction
  50 //
  51 #define IA32_PAE_PDPTE_ATTRIBUTE_BITS    (IA32_PG_P)
  52
  53 #define PAGE_PROGATE_BITS           (IA32_PG_NX | PAGE_ATTRIBUTE_BITS)
  54
  55 #define PAGING_4K_MASK  0xFFF
  56 #define PAGING_2M_MASK  0x1FFFFF
  57 #define PAGING_1G_MASK  0x3FFFFFFF
  58
  59 #define PAGING_PAE_INDEX_MASK  0x1FF
  60
  61 #define PAGING_4K_ADDRESS_MASK_64 0x000FFFFFFFFFF000ull
  62 #define PAGING_2M_ADDRESS_MASK_64 0x000FFFFFFFE00000ull
  63 #define PAGING_1G_ADDRESS_MASK_64 0x000FFFFFC0000000ull
  64
  65 typedef enum {
  66   PageNone,
  67   Page4K,
  68   Page2M,
  69   Page1G,
  70 } PAGE_ATTRIBUTE;
  71
  72 typedef struct {
  73   PAGE_ATTRIBUTE   Attribute;
  74   UINT64           Length;
  75   UINT64           AddressMask;
  76 } PAGE_ATTRIBUTE_TABLE;
  77
  78 typedef enum {
  79   PageActionAssign,
  80   PageActionSet,
  81   PageActionClear,
  82 } PAGE_ACTION;
  83
  84 PAGE_ATTRIBUTE_TABLE mPageAttributeTable[] = {
  85   {Page4K,  SIZE_4KB, PAGING_4K_ADDRESS_MASK_64},
  86   {Page2M,  SIZE_2MB, PAGING_2M_ADDRESS_MASK_64},
  87   {Page1G,  SIZE_1GB, PAGING_1G_ADDRESS_MASK_64},
  88 };
  89
  90 PAGE_TABLE_POOL   *mPageTablePool = NULL;
  91
  92 /**
  93   Enable write protection function for AP.
  94
  95   @param[in,out] Buffer  The pointer to private data buffer.
  96 **/
  97 VOID
  98 EFIAPI
  99 SyncCpuEnableWriteProtection (
 100   IN OUT VOID *Buffer
 101   )
 102 {
 103   AsmWriteCr0 (AsmReadCr0 () | BIT16);
 104 }
 105
 106 /**
 107   CpuFlushTlb function for AP.
 108
 109   @param[in,out] Buffer  The pointer to private data buffer.
 110 **/
 111 VOID
 112 EFIAPI
 113 SyncCpuFlushTlb (
 114   IN OUT VOID *Buffer
 115   )
 116 {
 117   CpuFlushTlb();
 118 }
 119
 120 /**
 121   Sync memory page attributes for AP.
 122
 123   @param[in] Procedure            A pointer to the function to be run on enabled APs of
 124                                   the system.
 125 **/
 126 VOID
 127 SyncMemoryPageAttributesAp (
 128   IN EFI_AP_PROCEDURE            Procedure
 129   )
 130 {
 131   EFI_STATUS                Status;
 132   EFI_MP_SERVICES_PROTOCOL  *MpService;
 133
 134   Status = gBS->LocateProtocol (
 135                   &gEfiMpServiceProtocolGuid,
 136                   NULL,
 137                   (VOID **)&MpService
 138                   );
 139   //
 140   // Synchronize the update with all APs
 141   //
 142   if (!EFI_ERROR (Status)) {
 143     Status = MpService->StartupAllAPs (
 144                           MpService,          // This
 145                           Procedure,          // Procedure
 146                           FALSE,              // SingleThread
 147                           NULL,               // WaitEvent
 148                           0,                  // TimeoutInMicrosecsond
 149                           NULL,               // ProcedureArgument
 150                           NULL                // FailedCpuList
 151                           );
 152     ASSERT (Status == EFI_SUCCESS || Status == EFI_NOT_STARTED || Status == EFI_NOT_READY);
 153   }
 154 }
 155
 156 /**
 157   Return current paging context.
 158
 159   @param[in,out]  PagingContext     The paging context.
 160 **/
 161 VOID
 162 GetCurrentPagingContext (
 163   IN OUT PAGE_TABLE_LIB_PAGING_CONTEXT     *PagingContext
 164   )
 165 {
 166   UINT32                         RegEax;
 167   UINT32                         RegEdx;
 168
 169   ZeroMem(PagingContext, sizeof(*PagingContext));
 170   if (sizeof(UINTN) == sizeof(UINT64)) {
 171     PagingContext->MachineType = IMAGE_FILE_MACHINE_X64;
 172   } else {
 173     PagingContext->MachineType = IMAGE_FILE_MACHINE_I386;
 174   }
 175   if ((AsmReadCr0 () & BIT31) != 0) {
 176     PagingContext->ContextData.X64.PageTableBase = (AsmReadCr3 () & PAGING_4K_ADDRESS_MASK_64);
 177   } else {
 178     PagingContext->ContextData.X64.PageTableBase = 0;
 179   }
 180
 181   if ((AsmReadCr4 () & BIT4) != 0) {
 182     PagingContext->ContextData.Ia32.Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PSE;
 183   }
 184   if ((AsmReadCr4 () & BIT5) != 0) {
 185     PagingContext->ContextData.Ia32.Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE;
 186   }
 187   if ((AsmReadCr0 () & BIT16) != 0) {
 188     PagingContext->ContextData.Ia32.Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_WP_ENABLE;
 189   }
 190
 191   AsmCpuid (0x80000000, &RegEax, NULL, NULL, NULL);
 192   if (RegEax > 0x80000000) {
 193     AsmCpuid (0x80000001, NULL, NULL, NULL, &RegEdx);
 194     if ((RegEdx & BIT20) != 0) {
 195       // XD supported
 196       if ((AsmReadMsr64 (0xC0000080) & BIT11) != 0) {
 197         // XD activated
 198         PagingContext->ContextData.Ia32.Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_XD_ACTIVATED;
 199       }
 200     }
 201     if ((RegEdx & BIT26) != 0) {
 202       PagingContext->ContextData.Ia32.Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAGE_1G_SUPPORT;
 203     }
 204   }
 205 }
 206
 207 /**
 208   Return length according to page attributes.
 209
 210   @param[in]  PageAttributes   The page attribute of the page entry.
 211
 212   @return The length of page entry.
 213 **/
 214 UINTN
 215 PageAttributeToLength (
 216   IN PAGE_ATTRIBUTE  PageAttribute
 217   )
 218 {
 219   UINTN  Index;
 220   for (Index = 0; Index < sizeof(mPageAttributeTable)/sizeof(mPageAttributeTable[0]); Index++) {
 221     if (PageAttribute == mPageAttributeTable[Index].Attribute) {
 222       return (UINTN)mPageAttributeTable[Index].Length;
 223     }
 224   }
 225   return 0;
 226 }
 227
 228 /**
 229   Return address mask according to page attributes.
 230
 231   @param[in]  PageAttributes   The page attribute of the page entry.
 232
 233   @return The address mask of page entry.
 234 **/
 235 UINTN
 236 PageAttributeToMask (
 237   IN PAGE_ATTRIBUTE  PageAttribute
 238   )
 239 {
 240   UINTN  Index;
 241   for (Index = 0; Index < sizeof(mPageAttributeTable)/sizeof(mPageAttributeTable[0]); Index++) {
 242     if (PageAttribute == mPageAttributeTable[Index].Attribute) {
 243       return (UINTN)mPageAttributeTable[Index].AddressMask;
 244     }
 245   }
 246   return 0;
 247 }
 248
 249 /**
 250   Return page table entry to match the address.
 251
 252   @param[in]  PagingContext     The paging context.
 253   @param[in]  Address           The address to be checked.
 254   @param[out] PageAttributes    The page attribute of the page entry.
 255
 256   @return The page entry.
 257 **/
 258 VOID *
 259 GetPageTableEntry (
 260   IN  PAGE_TABLE_LIB_PAGING_CONTEXT     *PagingContext,
 261   IN  PHYSICAL_ADDRESS                  Address,
 262   OUT PAGE_ATTRIBUTE                    *PageAttribute
 263   )
 264 {
 265   UINTN                 Index1;
 266   UINTN                 Index2;
 267   UINTN                 Index3;
 268   UINTN                 Index4;
 269   UINT64                *L1PageTable;
 270   UINT64                *L2PageTable;
 271   UINT64                *L3PageTable;
 272   UINT64                *L4PageTable;
 273   UINT64                AddressEncMask;
 274
 275   ASSERT (PagingContext != NULL);
 276
 277   Index4 = ((UINTN)RShiftU64 (Address, 39)) & PAGING_PAE_INDEX_MASK;
 278   Index3 = ((UINTN)Address >> 30) & PAGING_PAE_INDEX_MASK;
 279   Index2 = ((UINTN)Address >> 21) & PAGING_PAE_INDEX_MASK;
 280   Index1 = ((UINTN)Address >> 12) & PAGING_PAE_INDEX_MASK;
 281
 282   // Make sure AddressEncMask is contained to smallest supported address field.
 283   //
 284   AddressEncMask = PcdGet64 (PcdPteMemoryEncryptionAddressOrMask) & PAGING_1G_ADDRESS_MASK_64;
 285
 286   if (PagingContext->MachineType == IMAGE_FILE_MACHINE_X64) {
 287     L4PageTable = (UINT64 *)(UINTN)PagingContext->ContextData.X64.PageTableBase;
 288     if (L4PageTable[Index4] == 0) {
 289       *PageAttribute = PageNone;
 290       return NULL;
 291     }
 292
 293     L3PageTable = (UINT64 *)(UINTN)(L4PageTable[Index4] & ~AddressEncMask & PAGING_4K_ADDRESS_MASK_64);
 294   } else {
 295     ASSERT((PagingContext->ContextData.Ia32.Attributes & PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE) != 0);
 296     L3PageTable = (UINT64 *)(UINTN)PagingContext->ContextData.Ia32.PageTableBase;
 297   }
 298   if (L3PageTable[Index3] == 0) {
 299     *PageAttribute = PageNone;
 300     return NULL;
 301   }
 302   if ((L3PageTable[Index3] & IA32_PG_PS) != 0) {
 303     // 1G
 304     *PageAttribute = Page1G;
 305     return &L3PageTable[Index3];
 306   }
 307
 308   L2PageTable = (UINT64 *)(UINTN)(L3PageTable[Index3] & ~AddressEncMask & PAGING_4K_ADDRESS_MASK_64);
 309   if (L2PageTable[Index2] == 0) {
 310     *PageAttribute = PageNone;
 311     return NULL;
 312   }
 313   if ((L2PageTable[Index2] & IA32_PG_PS) != 0) {
 314     // 2M
 315     *PageAttribute = Page2M;
 316     return &L2PageTable[Index2];
 317   }
 318
 319   // 4k
 320   L1PageTable = (UINT64 *)(UINTN)(L2PageTable[Index2] & ~AddressEncMask & PAGING_4K_ADDRESS_MASK_64);
 321   if ((L1PageTable[Index1] == 0) && (Address != 0)) {
 322     *PageAttribute = PageNone;
 323     return NULL;
 324   }
 325   *PageAttribute = Page4K;
 326   return &L1PageTable[Index1];
 327 }
 328
 329 /**
 330   Return memory attributes of page entry.
 331
 332   @param[in]  PageEntry        The page entry.
 333
 334   @return Memory attributes of page entry.
 335 **/
 336 UINT64
 337 GetAttributesFromPageEntry (
 338   IN  UINT64                            *PageEntry
 339   )
 340 {
 341   UINT64  Attributes;
 342   Attributes = 0;
 343   if ((*PageEntry & IA32_PG_P) == 0) {
 344     Attributes |= EFI_MEMORY_RP;
 345   }
 346   if ((*PageEntry & IA32_PG_RW) == 0) {
 347     Attributes |= EFI_MEMORY_RO;
 348   }
 349   if ((*PageEntry & IA32_PG_NX) != 0) {
 350     Attributes |= EFI_MEMORY_XP;
 351   }
 352   return Attributes;
 353 }
 354
 355 /**
 356   Modify memory attributes of page entry.
 357
 358   @param[in]  PagingContext    The paging context.
 359   @param[in]  PageEntry        The page entry.
 360   @param[in]  Attributes       The bit mask of attributes to modify for the memory region.
 361   @param[in]  PageAction       The page action.
 362   @param[out] IsModified       TRUE means page table modified. FALSE means page table not modified.
 363 **/
 364 VOID
 365 ConvertPageEntryAttribute (
 366   IN  PAGE_TABLE_LIB_PAGING_CONTEXT     *PagingContext,
 367   IN  UINT64                            *PageEntry,
 368   IN  UINT64                            Attributes,
 369   IN  PAGE_ACTION                       PageAction,
 370   OUT BOOLEAN                           *IsModified
 371   )
 372 {
 373   UINT64  CurrentPageEntry;
 374   UINT64  NewPageEntry;
 375
 376   CurrentPageEntry = *PageEntry;
 377   NewPageEntry = CurrentPageEntry;
 378   if ((Attributes & EFI_MEMORY_RP) != 0) {
 379     switch (PageAction) {
 380     case PageActionAssign:
 381     case PageActionSet:
 382       NewPageEntry &= ~(UINT64)IA32_PG_P;
 383       break;
 384     case PageActionClear:
 385       NewPageEntry |= IA32_PG_P;
 386       break;
 387     }
 388   } else {
 389     switch (PageAction) {
 390     case PageActionAssign:
 391       NewPageEntry |= IA32_PG_P;
 392       break;
 393     case PageActionSet:
 394     case PageActionClear:
 395       break;
 396     }
 397   }
 398   if ((Attributes & EFI_MEMORY_RO) != 0) {
 399     switch (PageAction) {
 400     case PageActionAssign:
 401     case PageActionSet:
 402       NewPageEntry &= ~(UINT64)IA32_PG_RW;
 403       break;
 404     case PageActionClear:
 405       NewPageEntry |= IA32_PG_RW;
 406       break;
 407     }
 408   } else {
 409     switch (PageAction) {
 410     case PageActionAssign:
 411       NewPageEntry |= IA32_PG_RW;
 412       break;
 413     case PageActionSet:
 414     case PageActionClear:
 415       break;
 416     }
 417   }
 418   if ((PagingContext->ContextData.Ia32.Attributes & PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_XD_ACTIVATED) != 0) {
 419     if ((Attributes & EFI_MEMORY_XP) != 0) {
 420       switch (PageAction) {
 421       case PageActionAssign:
 422       case PageActionSet:
 423         NewPageEntry |= IA32_PG_NX;
 424         break;
 425       case PageActionClear:
 426         NewPageEntry &= ~IA32_PG_NX;
 427         break;
 428       }
 429     } else {
 430       switch (PageAction) {
 431       case PageActionAssign:
 432         NewPageEntry &= ~IA32_PG_NX;
 433         break;
 434       case PageActionSet:
 435       case PageActionClear:
 436         break;
 437       }
 438     }
 439   }
 440   *PageEntry = NewPageEntry;
 441   if (CurrentPageEntry != NewPageEntry) {
 442     *IsModified = TRUE;
 443     DEBUG ((DEBUG_VERBOSE, "ConvertPageEntryAttribute 0x%lx", CurrentPageEntry));
 444     DEBUG ((DEBUG_VERBOSE, "->0x%lx\n", NewPageEntry));
 445   } else {
 446     *IsModified = FALSE;
 447   }
 448 }
 449
 450 /**
 451   This function returns if there is need to split page entry.
 452
 453   @param[in]  BaseAddress      The base address to be checked.
 454   @param[in]  Length           The length to be checked.
 455   @param[in]  PageEntry        The page entry to be checked.
 456   @param[in]  PageAttribute    The page attribute of the page entry.
 457
 458   @retval SplitAttributes on if there is need to split page entry.
 459 **/
 460 PAGE_ATTRIBUTE
 461 NeedSplitPage (
 462   IN  PHYSICAL_ADDRESS                  BaseAddress,
 463   IN  UINT64                            Length,
 464   IN  UINT64                            *PageEntry,
 465   IN  PAGE_ATTRIBUTE                    PageAttribute
 466   )
 467 {
 468   UINT64                PageEntryLength;
 469
 470   PageEntryLength = PageAttributeToLength (PageAttribute);
 471
 472   if (((BaseAddress & (PageEntryLength - 1)) == 0) && (Length >= PageEntryLength)) {
 473     return PageNone;
 474   }
 475
 476   if (((BaseAddress & PAGING_2M_MASK) != 0) || (Length < SIZE_2MB)) {
 477     return Page4K;
 478   }
 479
 480   return Page2M;
 481 }
 482
 483 /**
 484   This function splits one page entry to small page entries.
 485
 486   @param[in]  PageEntry         The page entry to be splitted.
 487   @param[in]  PageAttribute     The page attribute of the page entry.
 488   @param[in]  SplitAttribute    How to split the page entry.
 489   @param[in]  AllocatePagesFunc If page split is needed, this function is used to allocate more pages.
 490
 491   @retval RETURN_SUCCESS            The page entry is splitted.
 492   @retval RETURN_UNSUPPORTED        The page entry does not support to be splitted.
 493   @retval RETURN_OUT_OF_RESOURCES   No resource to split page entry.
 494 **/
 495 RETURN_STATUS
 496 SplitPage (
 497   IN  UINT64                            *PageEntry,
 498   IN  PAGE_ATTRIBUTE                    PageAttribute,
 499   IN  PAGE_ATTRIBUTE                    SplitAttribute,
 500   IN  PAGE_TABLE_LIB_ALLOCATE_PAGES     AllocatePagesFunc
 501   )
 502 {
 503   UINT64   BaseAddress;
 504   UINT64   *NewPageEntry;
 505   UINTN    Index;
 506   UINT64   AddressEncMask;
 507
 508   ASSERT (PageAttribute == Page2M || PageAttribute == Page1G);
 509
 510   ASSERT (AllocatePagesFunc != NULL);
 511
 512   // Make sure AddressEncMask is contained to smallest supported address field.
 513   //
 514   AddressEncMask = PcdGet64 (PcdPteMemoryEncryptionAddressOrMask) & PAGING_1G_ADDRESS_MASK_64;
 515
 516   if (PageAttribute == Page2M) {
 517     //
 518     // Split 2M to 4K
 519     //
 520     ASSERT (SplitAttribute == Page4K);
 521     if (SplitAttribute == Page4K) {
 522       NewPageEntry = AllocatePagesFunc (1);
 523       DEBUG ((DEBUG_INFO, "Split - 0x%x\n", NewPageEntry));
 524       if (NewPageEntry == NULL) {
 525         return RETURN_OUT_OF_RESOURCES;
 526       }
 527       BaseAddress = *PageEntry & ~AddressEncMask & PAGING_2M_ADDRESS_MASK_64;
 528       for (Index = 0; Index < SIZE_4KB / sizeof(UINT64); Index++) {
 529         NewPageEntry[Index] = (BaseAddress + SIZE_4KB * Index) | AddressEncMask | ((*PageEntry) & PAGE_PROGATE_BITS);
 530       }
 531       (*PageEntry) = (UINT64)(UINTN)NewPageEntry | AddressEncMask | ((*PageEntry) & PAGE_PROGATE_BITS);
 532       return RETURN_SUCCESS;
 533     } else {
 534       return RETURN_UNSUPPORTED;
 535     }
 536   } else if (PageAttribute == Page1G) {
 537     //
 538     // Split 1G to 2M
 539     // No need support 1G->4K directly, we should use 1G->2M, then 2M->4K to get more compact page table.
 540     //
 541     ASSERT (SplitAttribute == Page2M || SplitAttribute == Page4K);
 542     if ((SplitAttribute == Page2M || SplitAttribute == Page4K)) {
 543       NewPageEntry = AllocatePagesFunc (1);
 544       DEBUG ((DEBUG_INFO, "Split - 0x%x\n", NewPageEntry));
 545       if (NewPageEntry == NULL) {
 546         return RETURN_OUT_OF_RESOURCES;
 547       }
 548       BaseAddress = *PageEntry & ~AddressEncMask  & PAGING_1G_ADDRESS_MASK_64;
 549       for (Index = 0; Index < SIZE_4KB / sizeof(UINT64); Index++) {
 550         NewPageEntry[Index] = (BaseAddress + SIZE_2MB * Index) | AddressEncMask | IA32_PG_PS | ((*PageEntry) & PAGE_PROGATE_BITS);
 551       }
 552       (*PageEntry) = (UINT64)(UINTN)NewPageEntry | AddressEncMask | ((*PageEntry) & PAGE_PROGATE_BITS);
 553       return RETURN_SUCCESS;
 554     } else {
 555       return RETURN_UNSUPPORTED;
 556     }
 557   } else {
 558     return RETURN_UNSUPPORTED;
 559   }
 560 }
 561
 562 /**
 563  Check the WP status in CR0 register. This bit is used to lock or unlock write
 564  access to pages marked as read-only.
 565
 566   @retval TRUE    Write protection is enabled.
 567   @retval FALSE   Write protection is disabled.
 568 **/
 569 BOOLEAN
 570 IsReadOnlyPageWriteProtected (
 571   VOID
 572   )
 573 {
 574   return ((AsmReadCr0 () & BIT16) != 0);
 575 }
 576
 577 /**
 578   Disable write protection function for AP.
 579
 580   @param[in,out] Buffer  The pointer to private data buffer.
 581 **/
 582 VOID
 583 EFIAPI
 584 SyncCpuDisableWriteProtection (
 585   IN OUT VOID *Buffer
 586   )
 587 {
 588   AsmWriteCr0 (AsmReadCr0() & ~BIT16);
 589 }
 590
 591 /**
 592  Disable Write Protect on pages marked as read-only.
 593 **/
 594 VOID
 595 DisableReadOnlyPageWriteProtect (
 596   VOID
 597   )
 598 {
 599   AsmWriteCr0 (AsmReadCr0() & ~BIT16);
 600   SyncMemoryPageAttributesAp (SyncCpuDisableWriteProtection);
 601 }
 602
 603 /**
 604  Enable Write Protect on pages marked as read-only.
 605 **/
 606 VOID
 607 EnableReadOnlyPageWriteProtect (
 608   VOID
 609   )
 610 {
 611   AsmWriteCr0 (AsmReadCr0() | BIT16);
 612   SyncMemoryPageAttributesAp (SyncCpuEnableWriteProtection);
 613 }
 614
 615 /**
 616   This function modifies the page attributes for the memory region specified by BaseAddress and
 617   Length from their current attributes to the attributes specified by Attributes.
 618
 619   Caller should make sure BaseAddress and Length is at page boundary.
 620
 621   @param[in]  PagingContext     The paging context. NULL means get page table from current CPU context.
 622   @param[in]  BaseAddress       The physical address that is the start address of a memory region.
 623   @param[in]  Length            The size in bytes of the memory region.
 624   @param[in]  Attributes        The bit mask of attributes to modify for the memory region.
 625   @param[in]  PageAction        The page action.
 626   @param[in]  AllocatePagesFunc If page split is needed, this function is used to allocate more pages.
 627                                 NULL mean page split is unsupported.
 628   @param[out] IsSplitted        TRUE means page table splitted. FALSE means page table not splitted.
 629   @param[out] IsModified        TRUE means page table modified. FALSE means page table not modified.
 630
 631   @retval RETURN_SUCCESS           The attributes were modified for the memory region.
 632   @retval RETURN_ACCESS_DENIED     The attributes for the memory resource range specified by
 633                                    BaseAddress and Length cannot be modified.
 634   @retval RETURN_INVALID_PARAMETER Length is zero.
 635                                    Attributes specified an illegal combination of attributes that
 636                                    cannot be set together.
 637   @retval RETURN_OUT_OF_RESOURCES  There are not enough system resources to modify the attributes of
 638                                    the memory resource range.
 639   @retval RETURN_UNSUPPORTED       The processor does not support one or more bytes of the memory
 640                                    resource range specified by BaseAddress and Length.
 641                                    The bit mask of attributes is not support for the memory resource
 642                                    range specified by BaseAddress and Length.
 643 **/
 644 RETURN_STATUS
 645 ConvertMemoryPageAttributes (
 646   IN  PAGE_TABLE_LIB_PAGING_CONTEXT     *PagingContext OPTIONAL,
 647   IN  PHYSICAL_ADDRESS                  BaseAddress,
 648   IN  UINT64                            Length,
 649   IN  UINT64                            Attributes,
 650   IN  PAGE_ACTION                       PageAction,
 651   IN  PAGE_TABLE_LIB_ALLOCATE_PAGES     AllocatePagesFunc OPTIONAL,
 652   OUT BOOLEAN                           *IsSplitted,  OPTIONAL
 653   OUT BOOLEAN                           *IsModified   OPTIONAL
 654   )
 655 {
 656   PAGE_TABLE_LIB_PAGING_CONTEXT     CurrentPagingContext;
 657   UINT64                            *PageEntry;
 658   PAGE_ATTRIBUTE                    PageAttribute;
 659   UINTN                             PageEntryLength;
 660   PAGE_ATTRIBUTE                    SplitAttribute;
 661   RETURN_STATUS                     Status;
 662   BOOLEAN                           IsEntryModified;
 663   BOOLEAN                           IsWpEnabled;
 664
 665   if ((BaseAddress & (SIZE_4KB - 1)) != 0) {
 666     DEBUG ((DEBUG_ERROR, "BaseAddress(0x%lx) is not aligned!\n", BaseAddress));
 667     return EFI_UNSUPPORTED;
 668   }
 669   if ((Length & (SIZE_4KB - 1)) != 0) {
 670     DEBUG ((DEBUG_ERROR, "Length(0x%lx) is not aligned!\n", Length));
 671     return EFI_UNSUPPORTED;
 672   }
 673   if (Length == 0) {
 674     DEBUG ((DEBUG_ERROR, "Length is 0!\n"));
 675     return RETURN_INVALID_PARAMETER;
 676   }
 677
 678   if ((Attributes & ~(EFI_MEMORY_RP | EFI_MEMORY_RO | EFI_MEMORY_XP)) != 0) {
 679     DEBUG ((DEBUG_ERROR, "Attributes(0x%lx) has unsupported bit\n", Attributes));
 680     return EFI_UNSUPPORTED;
 681   }
 682
 683   if (PagingContext == NULL) {
 684     GetCurrentPagingContext (&CurrentPagingContext);
 685   } else {
 686     CopyMem (&CurrentPagingContext, PagingContext, sizeof(CurrentPagingContext));
 687   }
 688   switch(CurrentPagingContext.MachineType) {
 689   case IMAGE_FILE_MACHINE_I386:
 690     if (CurrentPagingContext.ContextData.Ia32.PageTableBase == 0) {
 691       if (Attributes == 0) {
 692         return EFI_SUCCESS;
 693       } else {
 694         DEBUG ((DEBUG_ERROR, "PageTable is 0!\n"));
 695         return EFI_UNSUPPORTED;
 696       }
 697     }
 698     if ((CurrentPagingContext.ContextData.Ia32.Attributes & PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE) == 0) {
 699       DEBUG ((DEBUG_ERROR, "Non-PAE Paging!\n"));
 700       return EFI_UNSUPPORTED;
 701     }
 702     break;
 703   case IMAGE_FILE_MACHINE_X64:
 704     ASSERT (CurrentPagingContext.ContextData.X64.PageTableBase != 0);
 705     break;
 706   default:
 707     ASSERT(FALSE);
 708     return EFI_UNSUPPORTED;
 709     break;
 710   }
 711
 712 //  DEBUG ((DEBUG_ERROR, "ConvertMemoryPageAttributes(%x) - %016lx, %016lx, %02lx\n", IsSet, BaseAddress, Length, Attributes));
 713
 714   if (IsSplitted != NULL) {
 715     *IsSplitted = FALSE;
 716   }
 717   if (IsModified != NULL) {
 718     *IsModified = FALSE;
 719   }
 720   if (AllocatePagesFunc == NULL) {
 721     AllocatePagesFunc = AllocatePageTableMemory;
 722   }
 723
 724   //
 725   // Make sure that the page table is changeable.
 726   //
 727   IsWpEnabled = IsReadOnlyPageWriteProtected ();
 728   if (IsWpEnabled) {
 729     DisableReadOnlyPageWriteProtect ();
 730   }
 731
 732   //
 733   // Below logic is to check 2M/4K page to make sure we donot waist memory.
 734   //
 735   Status = EFI_SUCCESS;
 736   while (Length != 0) {
 737     PageEntry = GetPageTableEntry (&CurrentPagingContext, BaseAddress, &PageAttribute);
 738     if (PageEntry == NULL) {
 739       Status = RETURN_UNSUPPORTED;
 740       goto Done;
 741     }
 742     PageEntryLength = PageAttributeToLength (PageAttribute);
 743     SplitAttribute = NeedSplitPage (BaseAddress, Length, PageEntry, PageAttribute);
 744     if (SplitAttribute == PageNone) {
 745       ConvertPageEntryAttribute (&CurrentPagingContext, PageEntry, Attributes, PageAction, &IsEntryModified);
 746       if (IsEntryModified) {
 747         if (IsModified != NULL) {
 748           *IsModified = TRUE;
 749         }
 750       }
 751       //
 752       // Convert success, move to next
 753       //
 754       BaseAddress += PageEntryLength;
 755       Length -= PageEntryLength;
 756     } else {
 757       if (AllocatePagesFunc == NULL) {
 758         Status = RETURN_UNSUPPORTED;
 759         goto Done;
 760       }
 761       Status = SplitPage (PageEntry, PageAttribute, SplitAttribute, AllocatePagesFunc);
 762       if (RETURN_ERROR (Status)) {
 763         Status = RETURN_UNSUPPORTED;
 764         goto Done;
 765       }
 766       if (IsSplitted != NULL) {
 767         *IsSplitted = TRUE;
 768       }
 769       if (IsModified != NULL) {
 770         *IsModified = TRUE;
 771       }
 772       //
 773       // Just split current page
 774       // Convert success in next around
 775       //
 776     }
 777   }
 778
 779 Done:
 780   //
 781   // Restore page table write protection, if any.
 782   //
 783   if (IsWpEnabled) {
 784     EnableReadOnlyPageWriteProtect ();
 785   }
 786   return Status;
 787 }
 788
 789 /**
 790   This function assigns the page attributes for the memory region specified by BaseAddress and
 791   Length from their current attributes to the attributes specified by Attributes.
 792
 793   Caller should make sure BaseAddress and Length is at page boundary.
 794
 795   Caller need guarentee the TPL <= TPL_NOTIFY, if there is split page request.
 796
 797   @param[in]  PagingContext     The paging context. NULL means get page table from current CPU context.
 798   @param[in]  BaseAddress       The physical address that is the start address of a memory region.
 799   @param[in]  Length            The size in bytes of the memory region.
 800   @param[in]  Attributes        The bit mask of attributes to set for the memory region.
 801   @param[in]  AllocatePagesFunc If page split is needed, this function is used to allocate more pages.
 802                                 NULL mean page split is unsupported.
 803
 804   @retval RETURN_SUCCESS           The attributes were cleared for the memory region.
 805   @retval RETURN_ACCESS_DENIED     The attributes for the memory resource range specified by
 806                                    BaseAddress and Length cannot be modified.
 807   @retval RETURN_INVALID_PARAMETER Length is zero.
 808                                    Attributes specified an illegal combination of attributes that
 809                                    cannot be set together.
 810   @retval RETURN_OUT_OF_RESOURCES  There are not enough system resources to modify the attributes of
 811                                    the memory resource range.
 812   @retval RETURN_UNSUPPORTED       The processor does not support one or more bytes of the memory
 813                                    resource range specified by BaseAddress and Length.
 814                                    The bit mask of attributes is not support for the memory resource
 815                                    range specified by BaseAddress and Length.
 816 **/
 817 RETURN_STATUS
 818 EFIAPI
 819 AssignMemoryPageAttributes (
 820   IN  PAGE_TABLE_LIB_PAGING_CONTEXT     *PagingContext OPTIONAL,
 821   IN  PHYSICAL_ADDRESS                  BaseAddress,
 822   IN  UINT64                            Length,
 823   IN  UINT64                            Attributes,
 824   IN  PAGE_TABLE_LIB_ALLOCATE_PAGES     AllocatePagesFunc OPTIONAL
 825   )
 826 {
 827   RETURN_STATUS  Status;
 828   BOOLEAN        IsModified;
 829   BOOLEAN        IsSplitted;
 830
 831 //  DEBUG((DEBUG_INFO, "AssignMemoryPageAttributes: 0x%lx - 0x%lx (0x%lx)\n", BaseAddress, Length, Attributes));
 832   Status = ConvertMemoryPageAttributes (PagingContext, BaseAddress, Length, Attributes, PageActionAssign, AllocatePagesFunc, &IsSplitted, &IsModified);
 833   if (!EFI_ERROR(Status)) {
 834     if ((PagingContext == NULL) && IsModified) {
 835       //
 836       // Flush TLB as last step
 837       //
 838       CpuFlushTlb();
 839       SyncMemoryPageAttributesAp (SyncCpuFlushTlb);
 840     }
 841   }
 842
 843   return Status;
 844 }
 845
 846 /**
 847  Check if Execute Disable feature is enabled or not.
 848 **/
 849 BOOLEAN
 850 IsExecuteDisableEnabled (
 851   VOID
 852   )
 853 {
 854   MSR_CORE_IA32_EFER_REGISTER    MsrEfer;
 855
 856   MsrEfer.Uint64 = AsmReadMsr64 (MSR_IA32_EFER);
 857   return (MsrEfer.Bits.NXE == 1);
 858 }
 859
 860 /**
 861   Update GCD memory space attributes according to current page table setup.
 862 **/
 863 VOID
 864 RefreshGcdMemoryAttributesFromPaging (
 865   VOID
 866   )
 867 {
 868   EFI_STATUS                          Status;
 869   UINTN                               NumberOfDescriptors;
 870   EFI_GCD_MEMORY_SPACE_DESCRIPTOR     *MemorySpaceMap;
 871   PAGE_TABLE_LIB_PAGING_CONTEXT       PagingContext;
 872   PAGE_ATTRIBUTE                      PageAttribute;
 873   UINT64                              *PageEntry;
 874   UINT64                              PageLength;
 875   UINT64                              MemorySpaceLength;
 876   UINT64                              Length;
 877   UINT64                              BaseAddress;
 878   UINT64                              PageStartAddress;
 879   UINT64                              Attributes;
 880   UINT64                              Capabilities;
 881   UINT64                              NewAttributes;
 882   UINTN                               Index;
 883
 884   //
 885   // Assuming that memory space map returned is sorted already; otherwise sort
 886   // them in the order of lowest address to highest address.
 887   //
 888   Status = gDS->GetMemorySpaceMap (&NumberOfDescriptors, &MemorySpaceMap);
 889   ASSERT_EFI_ERROR (Status);
 890
 891   GetCurrentPagingContext (&PagingContext);
 892
 893   Attributes      = 0;
 894   NewAttributes   = 0;
 895   BaseAddress     = 0;
 896   PageLength      = 0;
 897
 898   if (IsExecuteDisableEnabled ()) {
 899     Capabilities = EFI_MEMORY_RO | EFI_MEMORY_RP | EFI_MEMORY_XP;
 900   } else {
 901     Capabilities = EFI_MEMORY_RO | EFI_MEMORY_RP;
 902   }
 903
 904   for (Index = 0; Index < NumberOfDescriptors; Index++) {
 905     if (MemorySpaceMap[Index].GcdMemoryType == EfiGcdMemoryTypeNonExistent) {
 906       continue;
 907     }
 908
 909     //
 910     // Sync the actual paging related capabilities back to GCD service first.
 911     // As a side effect (good one), this can also help to avoid unnecessary
 912     // memory map entries due to the different capabilities of the same type
 913     // memory, such as multiple RT_CODE and RT_DATA entries in memory map,
 914     // which could cause boot failure of some old Linux distro (before v4.3).
 915     //
 916     Status = gDS->SetMemorySpaceCapabilities (
 917                     MemorySpaceMap[Index].BaseAddress,
 918                     MemorySpaceMap[Index].Length,
 919                     MemorySpaceMap[Index].Capabilities | Capabilities
 920                     );
 921     if (EFI_ERROR (Status)) {
 922       //
 923       // If we cannot udpate the capabilities, we cannot update its
 924       // attributes either. So just simply skip current block of memory.
 925       //
 926       DEBUG ((
 927         DEBUG_WARN,
 928         "Failed to update capability: [%lu] %016lx - %016lx (%016lx -> %016lx)\r\n",
 929         (UINT64)Index, MemorySpaceMap[Index].BaseAddress,
 930         MemorySpaceMap[Index].BaseAddress + MemorySpaceMap[Index].Length - 1,
 931         MemorySpaceMap[Index].Capabilities,
 932         MemorySpaceMap[Index].Capabilities | Capabilities
 933         ));
 934       continue;
 935     }
 936
 937     if (MemorySpaceMap[Index].BaseAddress >= (BaseAddress + PageLength)) {
 938       //
 939       // Current memory space starts at a new page. Resetting PageLength will
 940       // trigger a retrieval of page attributes at new address.
 941       //
 942       PageLength = 0;
 943     } else {
 944       //
 945       // In case current memory space is not adjacent to last one
 946       //
 947       PageLength -= (MemorySpaceMap[Index].BaseAddress - BaseAddress);
 948     }
 949
 950     //
 951     // Sync actual page attributes to GCD
 952     //
 953     BaseAddress       = MemorySpaceMap[Index].BaseAddress;
 954     MemorySpaceLength = MemorySpaceMap[Index].Length;
 955     while (MemorySpaceLength > 0) {
 956       if (PageLength == 0) {
 957         PageEntry = GetPageTableEntry (&PagingContext, BaseAddress, &PageAttribute);
 958         if (PageEntry == NULL) {
 959           break;
 960         }
 961
 962         //
 963         // Note current memory space might start in the middle of a page
 964         //
 965         PageStartAddress  = (*PageEntry) & (UINT64)PageAttributeToMask(PageAttribute);
 966         PageLength        = PageAttributeToLength (PageAttribute) - (BaseAddress - PageStartAddress);
 967         Attributes        = GetAttributesFromPageEntry (PageEntry);
 968       }
 969
 970       Length = MIN (PageLength, MemorySpaceLength);
 971       if (Attributes != (MemorySpaceMap[Index].Attributes &
 972                          EFI_MEMORY_PAGETYPE_MASK)) {
 973         NewAttributes = (MemorySpaceMap[Index].Attributes &
 974                          ~EFI_MEMORY_PAGETYPE_MASK) | Attributes;
 975         Status = gDS->SetMemorySpaceAttributes (
 976                         BaseAddress,
 977                         Length,
 978                         NewAttributes
 979                         );
 980         ASSERT_EFI_ERROR (Status);
 981         DEBUG ((
 982           DEBUG_INFO,
 983           "Updated memory space attribute: [%lu] %016lx - %016lx (%016lx -> %016lx)\r\n",
 984           (UINT64)Index, BaseAddress, BaseAddress + Length - 1,
 985           MemorySpaceMap[Index].Attributes,
 986           NewAttributes
 987           ));
 988       }
 989
 990       PageLength        -= Length;
 991       MemorySpaceLength -= Length;
 992       BaseAddress       += Length;
 993     }
 994   }
 995
 996   FreePool (MemorySpaceMap);
 997 }
 998
 999 /**
1000   Initialize a buffer pool for page table use only.
1001
1002   To reduce the potential split operation on page table, the pages reserved for
1003   page table should be allocated in the times of PAGE_TABLE_POOL_UNIT_PAGES and
1004   at the boundary of PAGE_TABLE_POOL_ALIGNMENT. So the page pool is always
1005   initialized with number of pages greater than or equal to the given PoolPages.
1006
1007   Once the pages in the pool are used up, this method should be called again to
1008   reserve at least another PAGE_TABLE_POOL_UNIT_PAGES. Usually this won't happen
1009   often in practice.
1010
1011   @param[in] PoolPages      The least page number of the pool to be created.
1012
1013   @retval TRUE    The pool is initialized successfully.
1014   @retval FALSE   The memory is out of resource.
1015 **/
1016 BOOLEAN
1017 InitializePageTablePool (
1018   IN  UINTN                           PoolPages
1019   )
1020 {
1021   VOID                      *Buffer;
1022   BOOLEAN                   IsModified;
1023
1024   //
1025   // Always reserve at least PAGE_TABLE_POOL_UNIT_PAGES, including one page for
1026   // header.
1027   //
1028   PoolPages += 1;   // Add one page for header.
1029   PoolPages = ((PoolPages - 1) / PAGE_TABLE_POOL_UNIT_PAGES + 1) *
1030               PAGE_TABLE_POOL_UNIT_PAGES;
1031   Buffer = AllocateAlignedPages (PoolPages, PAGE_TABLE_POOL_ALIGNMENT);
1032   if (Buffer == NULL) {
1033     DEBUG ((DEBUG_ERROR, "ERROR: Out of aligned pages\r\n"));
1034     return FALSE;
1035   }
1036
1037   //
1038   // Link all pools into a list for easier track later.
1039   //
1040   if (mPageTablePool == NULL) {
1041     mPageTablePool = Buffer;
1042     mPageTablePool->NextPool = mPageTablePool;
1043   } else {
1044     ((PAGE_TABLE_POOL *)Buffer)->NextPool = mPageTablePool->NextPool;
1045     mPageTablePool->NextPool = Buffer;
1046     mPageTablePool = Buffer;
1047   }
1048
1049   //
1050   // Reserve one page for pool header.
1051   //
1052   mPageTablePool->FreePages  = PoolPages - 1;
1053   mPageTablePool->Offset = EFI_PAGES_TO_SIZE (1);
1054
1055   //
1056   // Mark the whole pool pages as read-only.
1057   //
1058   ConvertMemoryPageAttributes (
1059     NULL,
1060     (PHYSICAL_ADDRESS)(UINTN)Buffer,
1061     EFI_PAGES_TO_SIZE (PoolPages),
1062     EFI_MEMORY_RO,
1063     PageActionSet,
1064     AllocatePageTableMemory,
1065     NULL,
1066     &IsModified
1067     );
1068   ASSERT (IsModified == TRUE);
1069
1070   return TRUE;
1071 }
1072
1073 /**
1074   This API provides a way to allocate memory for page table.
1075
1076   This API can be called more than once to allocate memory for page tables.
1077
1078   Allocates the number of 4KB pages and returns a pointer to the allocated
1079   buffer. The buffer returned is aligned on a 4KB boundary.
1080
1081   If Pages is 0, then NULL is returned.
1082   If there is not enough memory remaining to satisfy the request, then NULL is
1083   returned.
1084
1085   @param  Pages                 The number of 4 KB pages to allocate.
1086
1087   @return A pointer to the allocated buffer or NULL if allocation fails.
1088
1089 **/
1090 VOID *
1091 EFIAPI
1092 AllocatePageTableMemory (
1093   IN UINTN           Pages
1094   )
1095 {
1096   VOID                            *Buffer;
1097
1098   if (Pages == 0) {
1099     return NULL;
1100   }
1101
1102   //
1103   // Renew the pool if necessary.
1104   //
1105   if (mPageTablePool == NULL ||
1106       Pages > mPageTablePool->FreePages) {
1107     if (!InitializePageTablePool (Pages)) {
1108       return NULL;
1109     }
1110   }
1111
1112   Buffer = (UINT8 *)mPageTablePool + mPageTablePool->Offset;
1113
1114   mPageTablePool->Offset     += EFI_PAGES_TO_SIZE (Pages);
1115   mPageTablePool->FreePages  -= Pages;
1116
1117   return Buffer;
1118 }
1119
1120 /**
1121   Initialize the Page Table lib.
1122 **/
1123 VOID
1124 InitializePageTableLib (
1125   VOID
1126   )
1127 {
1128   PAGE_TABLE_LIB_PAGING_CONTEXT     CurrentPagingContext;
1129
1130   GetCurrentPagingContext (&CurrentPagingContext);
1131
1132   //
1133   // Reserve memory of page tables for future uses, if paging is enabled.
1134   //
1135   if (CurrentPagingContext.ContextData.X64.PageTableBase != 0 &&
1136       (CurrentPagingContext.ContextData.Ia32.Attributes &
1137        PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE) != 0) {
1138     DisableReadOnlyPageWriteProtect ();
1139     InitializePageTablePool (1);
1140     EnableReadOnlyPageWriteProtect ();
1141   }
1142
1143   DEBUG ((DEBUG_INFO, "CurrentPagingContext:\n", CurrentPagingContext.MachineType));
1144   DEBUG ((DEBUG_INFO, "  MachineType   - 0x%x\n", CurrentPagingContext.MachineType));
1145   DEBUG ((DEBUG_INFO, "  PageTableBase - 0x%x\n", CurrentPagingContext.ContextData.X64.PageTableBase));
1146   DEBUG ((DEBUG_INFO, "  Attributes    - 0x%x\n", CurrentPagingContext.ContextData.X64.Attributes));
1147
1148   return ;
1149 }
1150