UefiCpuPkg/CpuDxe/CpuPageTable.c

   1 /** @file
   2   Page table management support.
   3
   4   Copyright (c) 2017, Intel Corporation. All rights reserved.<BR>
   5   Copyright (c) 2017, AMD Incorporated. All rights reserved.<BR>
   6
   7   This program and the accompanying materials
   8   are licensed and made available under the terms and conditions of the BSD License
   9   which accompanies this distribution.  The full text of the license may be found at
  10   http://opensource.org/licenses/bsd-license.php
  11
  12   THE PROGRAM IS DISTRIBUTED UNDER THE BSD LICENSE ON AN "AS IS" BASIS,
  13   WITHOUT WARRANTIES OR REPRESENTATIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED.
  14
  15 **/
  16
  17 #include <Base.h>
  18 #include <Uefi.h>
  19 #include <Library/BaseLib.h>
  20 #include <Library/CpuLib.h>
  21 #include <Library/BaseMemoryLib.h>
  22 #include <Library/MemoryAllocationLib.h>
  23 #include <Library/DebugLib.h>
  24 #include <Library/UefiBootServicesTableLib.h>
  25 #include <Protocol/MpService.h>
  26
  27 #include "CpuDxe.h"
  28 #include "CpuPageTable.h"
  29
  30 ///
  31 /// Page Table Entry
  32 ///
  33 #define IA32_PG_P                   BIT0
  34 #define IA32_PG_RW                  BIT1
  35 #define IA32_PG_U                   BIT2
  36 #define IA32_PG_WT                  BIT3
  37 #define IA32_PG_CD                  BIT4
  38 #define IA32_PG_A                   BIT5
  39 #define IA32_PG_D                   BIT6
  40 #define IA32_PG_PS                  BIT7
  41 #define IA32_PG_PAT_2M              BIT12
  42 #define IA32_PG_PAT_4K              IA32_PG_PS
  43 #define IA32_PG_PMNT                BIT62
  44 #define IA32_PG_NX                  BIT63
  45
  46 #define PAGE_ATTRIBUTE_BITS         (IA32_PG_D | IA32_PG_A | IA32_PG_U | IA32_PG_RW | IA32_PG_P)
  47 //
  48 // Bits 1, 2, 5, 6 are reserved in the IA32 PAE PDPTE
  49 // X64 PAE PDPTE does not have such restriction
  50 //
  51 #define IA32_PAE_PDPTE_ATTRIBUTE_BITS    (IA32_PG_P)
  52
  53 #define PAGE_PROGATE_BITS           (IA32_PG_NX | PAGE_ATTRIBUTE_BITS)
  54
  55 #define PAGING_4K_MASK  0xFFF
  56 #define PAGING_2M_MASK  0x1FFFFF
  57 #define PAGING_1G_MASK  0x3FFFFFFF
  58
  59 #define PAGING_PAE_INDEX_MASK  0x1FF
  60
  61 #define PAGING_4K_ADDRESS_MASK_64 0x000FFFFFFFFFF000ull
  62 #define PAGING_2M_ADDRESS_MASK_64 0x000FFFFFFFE00000ull
  63 #define PAGING_1G_ADDRESS_MASK_64 0x000FFFFFC0000000ull
  64
  65 typedef enum {
  66   PageNone,
  67   Page4K,
  68   Page2M,
  69   Page1G,
  70 } PAGE_ATTRIBUTE;
  71
  72 typedef struct {
  73   PAGE_ATTRIBUTE   Attribute;
  74   UINT64           Length;
  75   UINT64           AddressMask;
  76 } PAGE_ATTRIBUTE_TABLE;
  77
  78 typedef enum {
  79   PageActionAssign,
  80   PageActionSet,
  81   PageActionClear,
  82 } PAGE_ACTION;
  83
  84 PAGE_ATTRIBUTE_TABLE mPageAttributeTable[] = {
  85   {Page4K,  SIZE_4KB, PAGING_4K_ADDRESS_MASK_64},
  86   {Page2M,  SIZE_2MB, PAGING_2M_ADDRESS_MASK_64},
  87   {Page1G,  SIZE_1GB, PAGING_1G_ADDRESS_MASK_64},
  88 };
  89
  90 PAGE_TABLE_POOL   *mPageTablePool = NULL;
  91
  92 /**
  93   Return current paging context.
  94
  95   @param[in,out]  PagingContext     The paging context.
  96 **/
  97 VOID
  98 GetCurrentPagingContext (
  99   IN OUT PAGE_TABLE_LIB_PAGING_CONTEXT     *PagingContext
 100   )
 101 {
 102   UINT32                         RegEax;
 103   UINT32                         RegEdx;
 104
 105   ZeroMem(PagingContext, sizeof(*PagingContext));
 106   if (sizeof(UINTN) == sizeof(UINT64)) {
 107     PagingContext->MachineType = IMAGE_FILE_MACHINE_X64;
 108   } else {
 109     PagingContext->MachineType = IMAGE_FILE_MACHINE_I386;
 110   }
 111   if ((AsmReadCr0 () & BIT31) != 0) {
 112     PagingContext->ContextData.X64.PageTableBase = (AsmReadCr3 () & PAGING_4K_ADDRESS_MASK_64);
 113   } else {
 114     PagingContext->ContextData.X64.PageTableBase = 0;
 115   }
 116
 117   if ((AsmReadCr4 () & BIT4) != 0) {
 118     PagingContext->ContextData.Ia32.Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PSE;
 119   }
 120   if ((AsmReadCr4 () & BIT5) != 0) {
 121     PagingContext->ContextData.Ia32.Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE;
 122   }
 123   if ((AsmReadCr0 () & BIT16) != 0) {
 124     PagingContext->ContextData.Ia32.Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_WP_ENABLE;
 125   }
 126
 127   AsmCpuid (0x80000000, &RegEax, NULL, NULL, NULL);
 128   if (RegEax > 0x80000000) {
 129     AsmCpuid (0x80000001, NULL, NULL, NULL, &RegEdx);
 130     if ((RegEdx & BIT20) != 0) {
 131       // XD supported
 132       if ((AsmReadMsr64 (0xC0000080) & BIT11) != 0) {
 133         // XD activated
 134         PagingContext->ContextData.Ia32.Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_XD_ACTIVATED;
 135       }
 136     }
 137     if ((RegEdx & BIT26) != 0) {
 138       PagingContext->ContextData.Ia32.Attributes |= PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAGE_1G_SUPPORT;
 139     }
 140   }
 141 }
 142
 143 /**
 144   Return length according to page attributes.
 145
 146   @param[in]  PageAttributes   The page attribute of the page entry.
 147
 148   @return The length of page entry.
 149 **/
 150 UINTN
 151 PageAttributeToLength (
 152   IN PAGE_ATTRIBUTE  PageAttribute
 153   )
 154 {
 155   UINTN  Index;
 156   for (Index = 0; Index < sizeof(mPageAttributeTable)/sizeof(mPageAttributeTable[0]); Index++) {
 157     if (PageAttribute == mPageAttributeTable[Index].Attribute) {
 158       return (UINTN)mPageAttributeTable[Index].Length;
 159     }
 160   }
 161   return 0;
 162 }
 163
 164 /**
 165   Return address mask according to page attributes.
 166
 167   @param[in]  PageAttributes   The page attribute of the page entry.
 168
 169   @return The address mask of page entry.
 170 **/
 171 UINTN
 172 PageAttributeToMask (
 173   IN PAGE_ATTRIBUTE  PageAttribute
 174   )
 175 {
 176   UINTN  Index;
 177   for (Index = 0; Index < sizeof(mPageAttributeTable)/sizeof(mPageAttributeTable[0]); Index++) {
 178     if (PageAttribute == mPageAttributeTable[Index].Attribute) {
 179       return (UINTN)mPageAttributeTable[Index].AddressMask;
 180     }
 181   }
 182   return 0;
 183 }
 184
 185 /**
 186   Return page table entry to match the address.
 187
 188   @param[in]  PagingContext     The paging context.
 189   @param[in]  Address           The address to be checked.
 190   @param[out] PageAttributes    The page attribute of the page entry.
 191
 192   @return The page entry.
 193 **/
 194 VOID *
 195 GetPageTableEntry (
 196   IN  PAGE_TABLE_LIB_PAGING_CONTEXT     *PagingContext,
 197   IN  PHYSICAL_ADDRESS                  Address,
 198   OUT PAGE_ATTRIBUTE                    *PageAttribute
 199   )
 200 {
 201   UINTN                 Index1;
 202   UINTN                 Index2;
 203   UINTN                 Index3;
 204   UINTN                 Index4;
 205   UINT64                *L1PageTable;
 206   UINT64                *L2PageTable;
 207   UINT64                *L3PageTable;
 208   UINT64                *L4PageTable;
 209   UINT64                AddressEncMask;
 210
 211   ASSERT (PagingContext != NULL);
 212
 213   Index4 = ((UINTN)RShiftU64 (Address, 39)) & PAGING_PAE_INDEX_MASK;
 214   Index3 = ((UINTN)Address >> 30) & PAGING_PAE_INDEX_MASK;
 215   Index2 = ((UINTN)Address >> 21) & PAGING_PAE_INDEX_MASK;
 216   Index1 = ((UINTN)Address >> 12) & PAGING_PAE_INDEX_MASK;
 217
 218   // Make sure AddressEncMask is contained to smallest supported address field.
 219   //
 220   AddressEncMask = PcdGet64 (PcdPteMemoryEncryptionAddressOrMask) & PAGING_1G_ADDRESS_MASK_64;
 221
 222   if (PagingContext->MachineType == IMAGE_FILE_MACHINE_X64) {
 223     L4PageTable = (UINT64 *)(UINTN)PagingContext->ContextData.X64.PageTableBase;
 224     if (L4PageTable[Index4] == 0) {
 225       *PageAttribute = PageNone;
 226       return NULL;
 227     }
 228
 229     L3PageTable = (UINT64 *)(UINTN)(L4PageTable[Index4] & ~AddressEncMask & PAGING_4K_ADDRESS_MASK_64);
 230   } else {
 231     ASSERT((PagingContext->ContextData.Ia32.Attributes & PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE) != 0);
 232     L3PageTable = (UINT64 *)(UINTN)PagingContext->ContextData.Ia32.PageTableBase;
 233   }
 234   if (L3PageTable[Index3] == 0) {
 235     *PageAttribute = PageNone;
 236     return NULL;
 237   }
 238   if ((L3PageTable[Index3] & IA32_PG_PS) != 0) {
 239     // 1G
 240     *PageAttribute = Page1G;
 241     return &L3PageTable[Index3];
 242   }
 243
 244   L2PageTable = (UINT64 *)(UINTN)(L3PageTable[Index3] & ~AddressEncMask & PAGING_4K_ADDRESS_MASK_64);
 245   if (L2PageTable[Index2] == 0) {
 246     *PageAttribute = PageNone;
 247     return NULL;
 248   }
 249   if ((L2PageTable[Index2] & IA32_PG_PS) != 0) {
 250     // 2M
 251     *PageAttribute = Page2M;
 252     return &L2PageTable[Index2];
 253   }
 254
 255   // 4k
 256   L1PageTable = (UINT64 *)(UINTN)(L2PageTable[Index2] & ~AddressEncMask & PAGING_4K_ADDRESS_MASK_64);
 257   if ((L1PageTable[Index1] == 0) && (Address != 0)) {
 258     *PageAttribute = PageNone;
 259     return NULL;
 260   }
 261   *PageAttribute = Page4K;
 262   return &L1PageTable[Index1];
 263 }
 264
 265 /**
 266   Return memory attributes of page entry.
 267
 268   @param[in]  PageEntry        The page entry.
 269
 270   @return Memory attributes of page entry.
 271 **/
 272 UINT64
 273 GetAttributesFromPageEntry (
 274   IN  UINT64                            *PageEntry
 275   )
 276 {
 277   UINT64  Attributes;
 278   Attributes = 0;
 279   if ((*PageEntry & IA32_PG_P) == 0) {
 280     Attributes |= EFI_MEMORY_RP;
 281   }
 282   if ((*PageEntry & IA32_PG_RW) == 0) {
 283     Attributes |= EFI_MEMORY_RO;
 284   }
 285   if ((*PageEntry & IA32_PG_NX) != 0) {
 286     Attributes |= EFI_MEMORY_XP;
 287   }
 288   return Attributes;
 289 }
 290
 291 /**
 292   Modify memory attributes of page entry.
 293
 294   @param[in]  PagingContext    The paging context.
 295   @param[in]  PageEntry        The page entry.
 296   @param[in]  Attributes       The bit mask of attributes to modify for the memory region.
 297   @param[in]  PageAction       The page action.
 298   @param[out] IsModified       TRUE means page table modified. FALSE means page table not modified.
 299 **/
 300 VOID
 301 ConvertPageEntryAttribute (
 302   IN  PAGE_TABLE_LIB_PAGING_CONTEXT     *PagingContext,
 303   IN  UINT64                            *PageEntry,
 304   IN  UINT64                            Attributes,
 305   IN  PAGE_ACTION                       PageAction,
 306   OUT BOOLEAN                           *IsModified
 307   )
 308 {
 309   UINT64  CurrentPageEntry;
 310   UINT64  NewPageEntry;
 311
 312   CurrentPageEntry = *PageEntry;
 313   NewPageEntry = CurrentPageEntry;
 314   if ((Attributes & EFI_MEMORY_RP) != 0) {
 315     switch (PageAction) {
 316     case PageActionAssign:
 317     case PageActionSet:
 318       NewPageEntry &= ~(UINT64)IA32_PG_P;
 319       break;
 320     case PageActionClear:
 321       NewPageEntry |= IA32_PG_P;
 322       break;
 323     }
 324   } else {
 325     switch (PageAction) {
 326     case PageActionAssign:
 327       NewPageEntry |= IA32_PG_P;
 328       break;
 329     case PageActionSet:
 330     case PageActionClear:
 331       break;
 332     }
 333   }
 334   if ((Attributes & EFI_MEMORY_RO) != 0) {
 335     switch (PageAction) {
 336     case PageActionAssign:
 337     case PageActionSet:
 338       NewPageEntry &= ~(UINT64)IA32_PG_RW;
 339       break;
 340     case PageActionClear:
 341       NewPageEntry |= IA32_PG_RW;
 342       break;
 343     }
 344   } else {
 345     switch (PageAction) {
 346     case PageActionAssign:
 347       NewPageEntry |= IA32_PG_RW;
 348       break;
 349     case PageActionSet:
 350     case PageActionClear:
 351       break;
 352     }
 353   }
 354   if ((PagingContext->ContextData.Ia32.Attributes & PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_XD_ACTIVATED) != 0) {
 355     if ((Attributes & EFI_MEMORY_XP) != 0) {
 356       switch (PageAction) {
 357       case PageActionAssign:
 358       case PageActionSet:
 359         NewPageEntry |= IA32_PG_NX;
 360         break;
 361       case PageActionClear:
 362         NewPageEntry &= ~IA32_PG_NX;
 363         break;
 364       }
 365     } else {
 366       switch (PageAction) {
 367       case PageActionAssign:
 368         NewPageEntry &= ~IA32_PG_NX;
 369         break;
 370       case PageActionSet:
 371       case PageActionClear:
 372         break;
 373       }
 374     }
 375   }
 376   *PageEntry = NewPageEntry;
 377   if (CurrentPageEntry != NewPageEntry) {
 378     *IsModified = TRUE;
 379     DEBUG ((DEBUG_VERBOSE, "ConvertPageEntryAttribute 0x%lx", CurrentPageEntry));
 380     DEBUG ((DEBUG_VERBOSE, "->0x%lx\n", NewPageEntry));
 381   } else {
 382     *IsModified = FALSE;
 383   }
 384 }
 385
 386 /**
 387   This function returns if there is need to split page entry.
 388
 389   @param[in]  BaseAddress      The base address to be checked.
 390   @param[in]  Length           The length to be checked.
 391   @param[in]  PageEntry        The page entry to be checked.
 392   @param[in]  PageAttribute    The page attribute of the page entry.
 393
 394   @retval SplitAttributes on if there is need to split page entry.
 395 **/
 396 PAGE_ATTRIBUTE
 397 NeedSplitPage (
 398   IN  PHYSICAL_ADDRESS                  BaseAddress,
 399   IN  UINT64                            Length,
 400   IN  UINT64                            *PageEntry,
 401   IN  PAGE_ATTRIBUTE                    PageAttribute
 402   )
 403 {
 404   UINT64                PageEntryLength;
 405
 406   PageEntryLength = PageAttributeToLength (PageAttribute);
 407
 408   if (((BaseAddress & (PageEntryLength - 1)) == 0) && (Length >= PageEntryLength)) {
 409     return PageNone;
 410   }
 411
 412   if (((BaseAddress & PAGING_2M_MASK) != 0) || (Length < SIZE_2MB)) {
 413     return Page4K;
 414   }
 415
 416   return Page2M;
 417 }
 418
 419 /**
 420   This function splits one page entry to small page entries.
 421
 422   @param[in]  PageEntry         The page entry to be splitted.
 423   @param[in]  PageAttribute     The page attribute of the page entry.
 424   @param[in]  SplitAttribute    How to split the page entry.
 425   @param[in]  AllocatePagesFunc If page split is needed, this function is used to allocate more pages.
 426
 427   @retval RETURN_SUCCESS            The page entry is splitted.
 428   @retval RETURN_UNSUPPORTED        The page entry does not support to be splitted.
 429   @retval RETURN_OUT_OF_RESOURCES   No resource to split page entry.
 430 **/
 431 RETURN_STATUS
 432 SplitPage (
 433   IN  UINT64                            *PageEntry,
 434   IN  PAGE_ATTRIBUTE                    PageAttribute,
 435   IN  PAGE_ATTRIBUTE                    SplitAttribute,
 436   IN  PAGE_TABLE_LIB_ALLOCATE_PAGES     AllocatePagesFunc
 437   )
 438 {
 439   UINT64   BaseAddress;
 440   UINT64   *NewPageEntry;
 441   UINTN    Index;
 442   UINT64   AddressEncMask;
 443
 444   ASSERT (PageAttribute == Page2M || PageAttribute == Page1G);
 445
 446   ASSERT (AllocatePagesFunc != NULL);
 447
 448   // Make sure AddressEncMask is contained to smallest supported address field.
 449   //
 450   AddressEncMask = PcdGet64 (PcdPteMemoryEncryptionAddressOrMask) & PAGING_1G_ADDRESS_MASK_64;
 451
 452   if (PageAttribute == Page2M) {
 453     //
 454     // Split 2M to 4K
 455     //
 456     ASSERT (SplitAttribute == Page4K);
 457     if (SplitAttribute == Page4K) {
 458       NewPageEntry = AllocatePagesFunc (1);
 459       DEBUG ((DEBUG_INFO, "Split - 0x%x\n", NewPageEntry));
 460       if (NewPageEntry == NULL) {
 461         return RETURN_OUT_OF_RESOURCES;
 462       }
 463       BaseAddress = *PageEntry & ~AddressEncMask & PAGING_2M_ADDRESS_MASK_64;
 464       for (Index = 0; Index < SIZE_4KB / sizeof(UINT64); Index++) {
 465         NewPageEntry[Index] = (BaseAddress + SIZE_4KB * Index) | AddressEncMask | ((*PageEntry) & PAGE_PROGATE_BITS);
 466       }
 467       (*PageEntry) = (UINT64)(UINTN)NewPageEntry | AddressEncMask | ((*PageEntry) & PAGE_ATTRIBUTE_BITS);
 468       return RETURN_SUCCESS;
 469     } else {
 470       return RETURN_UNSUPPORTED;
 471     }
 472   } else if (PageAttribute == Page1G) {
 473     //
 474     // Split 1G to 2M
 475     // No need support 1G->4K directly, we should use 1G->2M, then 2M->4K to get more compact page table.
 476     //
 477     ASSERT (SplitAttribute == Page2M || SplitAttribute == Page4K);
 478     if ((SplitAttribute == Page2M || SplitAttribute == Page4K)) {
 479       NewPageEntry = AllocatePagesFunc (1);
 480       DEBUG ((DEBUG_INFO, "Split - 0x%x\n", NewPageEntry));
 481       if (NewPageEntry == NULL) {
 482         return RETURN_OUT_OF_RESOURCES;
 483       }
 484       BaseAddress = *PageEntry & ~AddressEncMask  & PAGING_1G_ADDRESS_MASK_64;
 485       for (Index = 0; Index < SIZE_4KB / sizeof(UINT64); Index++) {
 486         NewPageEntry[Index] = (BaseAddress + SIZE_2MB * Index) | AddressEncMask | IA32_PG_PS | ((*PageEntry) & PAGE_PROGATE_BITS);
 487       }
 488       (*PageEntry) = (UINT64)(UINTN)NewPageEntry | AddressEncMask | ((*PageEntry) & PAGE_ATTRIBUTE_BITS);
 489       return RETURN_SUCCESS;
 490     } else {
 491       return RETURN_UNSUPPORTED;
 492     }
 493   } else {
 494     return RETURN_UNSUPPORTED;
 495   }
 496 }
 497
 498 /**
 499  Check the WP status in CR0 register. This bit is used to lock or unlock write
 500  access to pages marked as read-only.
 501
 502   @retval TRUE    Write protection is enabled.
 503   @retval FALSE   Write protection is disabled.
 504 **/
 505 BOOLEAN
 506 IsReadOnlyPageWriteProtected (
 507   VOID
 508   )
 509 {
 510   return ((AsmReadCr0 () & BIT16) != 0);
 511 }
 512
 513 /**
 514  Disable Write Protect on pages marked as read-only.
 515 **/
 516 VOID
 517 DisableReadOnlyPageWriteProtect (
 518   VOID
 519   )
 520 {
 521   AsmWriteCr0 (AsmReadCr0() & ~BIT16);
 522 }
 523
 524 /**
 525  Enable Write Protect on pages marked as read-only.
 526 **/
 527 VOID
 528 EnableReadOnlyPageWriteProtect (
 529   VOID
 530   )
 531 {
 532   AsmWriteCr0 (AsmReadCr0() | BIT16);
 533 }
 534
 535 /**
 536   This function modifies the page attributes for the memory region specified by BaseAddress and
 537   Length from their current attributes to the attributes specified by Attributes.
 538
 539   Caller should make sure BaseAddress and Length is at page boundary.
 540
 541   @param[in]  PagingContext     The paging context. NULL means get page table from current CPU context.
 542   @param[in]  BaseAddress       The physical address that is the start address of a memory region.
 543   @param[in]  Length            The size in bytes of the memory region.
 544   @param[in]  Attributes        The bit mask of attributes to modify for the memory region.
 545   @param[in]  PageAction        The page action.
 546   @param[in]  AllocatePagesFunc If page split is needed, this function is used to allocate more pages.
 547                                 NULL mean page split is unsupported.
 548   @param[out] IsSplitted        TRUE means page table splitted. FALSE means page table not splitted.
 549   @param[out] IsModified        TRUE means page table modified. FALSE means page table not modified.
 550
 551   @retval RETURN_SUCCESS           The attributes were modified for the memory region.
 552   @retval RETURN_ACCESS_DENIED     The attributes for the memory resource range specified by
 553                                    BaseAddress and Length cannot be modified.
 554   @retval RETURN_INVALID_PARAMETER Length is zero.
 555                                    Attributes specified an illegal combination of attributes that
 556                                    cannot be set together.
 557   @retval RETURN_OUT_OF_RESOURCES  There are not enough system resources to modify the attributes of
 558                                    the memory resource range.
 559   @retval RETURN_UNSUPPORTED       The processor does not support one or more bytes of the memory
 560                                    resource range specified by BaseAddress and Length.
 561                                    The bit mask of attributes is not support for the memory resource
 562                                    range specified by BaseAddress and Length.
 563 **/
 564 RETURN_STATUS
 565 ConvertMemoryPageAttributes (
 566   IN  PAGE_TABLE_LIB_PAGING_CONTEXT     *PagingContext OPTIONAL,
 567   IN  PHYSICAL_ADDRESS                  BaseAddress,
 568   IN  UINT64                            Length,
 569   IN  UINT64                            Attributes,
 570   IN  PAGE_ACTION                       PageAction,
 571   IN  PAGE_TABLE_LIB_ALLOCATE_PAGES     AllocatePagesFunc OPTIONAL,
 572   OUT BOOLEAN                           *IsSplitted,  OPTIONAL
 573   OUT BOOLEAN                           *IsModified   OPTIONAL
 574   )
 575 {
 576   PAGE_TABLE_LIB_PAGING_CONTEXT     CurrentPagingContext;
 577   UINT64                            *PageEntry;
 578   PAGE_ATTRIBUTE                    PageAttribute;
 579   UINTN                             PageEntryLength;
 580   PAGE_ATTRIBUTE                    SplitAttribute;
 581   RETURN_STATUS                     Status;
 582   BOOLEAN                           IsEntryModified;
 583   BOOLEAN                           IsWpEnabled;
 584
 585   if ((BaseAddress & (SIZE_4KB - 1)) != 0) {
 586     DEBUG ((DEBUG_ERROR, "BaseAddress(0x%lx) is not aligned!\n", BaseAddress));
 587     return EFI_UNSUPPORTED;
 588   }
 589   if ((Length & (SIZE_4KB - 1)) != 0) {
 590     DEBUG ((DEBUG_ERROR, "Length(0x%lx) is not aligned!\n", Length));
 591     return EFI_UNSUPPORTED;
 592   }
 593   if (Length == 0) {
 594     DEBUG ((DEBUG_ERROR, "Length is 0!\n"));
 595     return RETURN_INVALID_PARAMETER;
 596   }
 597
 598   if ((Attributes & ~(EFI_MEMORY_RP | EFI_MEMORY_RO | EFI_MEMORY_XP)) != 0) {
 599     DEBUG ((DEBUG_ERROR, "Attributes(0x%lx) has unsupported bit\n", Attributes));
 600     return EFI_UNSUPPORTED;
 601   }
 602
 603   if (PagingContext == NULL) {
 604     GetCurrentPagingContext (&CurrentPagingContext);
 605   } else {
 606     CopyMem (&CurrentPagingContext, PagingContext, sizeof(CurrentPagingContext));
 607   }
 608   switch(CurrentPagingContext.MachineType) {
 609   case IMAGE_FILE_MACHINE_I386:
 610     if (CurrentPagingContext.ContextData.Ia32.PageTableBase == 0) {
 611       if (Attributes == 0) {
 612         return EFI_SUCCESS;
 613       } else {
 614         DEBUG ((DEBUG_ERROR, "PageTable is 0!\n"));
 615         return EFI_UNSUPPORTED;
 616       }
 617     }
 618     if ((CurrentPagingContext.ContextData.Ia32.Attributes & PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE) == 0) {
 619       DEBUG ((DEBUG_ERROR, "Non-PAE Paging!\n"));
 620       return EFI_UNSUPPORTED;
 621     }
 622     if ((BaseAddress + Length) > BASE_4GB) {
 623       DEBUG ((DEBUG_ERROR, "Beyond 4GB memory in 32-bit mode!\n"));
 624       return EFI_UNSUPPORTED;
 625     }
 626     break;
 627   case IMAGE_FILE_MACHINE_X64:
 628     ASSERT (CurrentPagingContext.ContextData.X64.PageTableBase != 0);
 629     break;
 630   default:
 631     ASSERT(FALSE);
 632     return EFI_UNSUPPORTED;
 633     break;
 634   }
 635
 636 //  DEBUG ((DEBUG_ERROR, "ConvertMemoryPageAttributes(%x) - %016lx, %016lx, %02lx\n", IsSet, BaseAddress, Length, Attributes));
 637
 638   if (IsSplitted != NULL) {
 639     *IsSplitted = FALSE;
 640   }
 641   if (IsModified != NULL) {
 642     *IsModified = FALSE;
 643   }
 644   if (AllocatePagesFunc == NULL) {
 645     AllocatePagesFunc = AllocatePageTableMemory;
 646   }
 647
 648   //
 649   // Make sure that the page table is changeable.
 650   //
 651   IsWpEnabled = IsReadOnlyPageWriteProtected ();
 652   if (IsWpEnabled) {
 653     DisableReadOnlyPageWriteProtect ();
 654   }
 655
 656   //
 657   // Below logic is to check 2M/4K page to make sure we donot waist memory.
 658   //
 659   Status = EFI_SUCCESS;
 660   while (Length != 0) {
 661     PageEntry = GetPageTableEntry (&CurrentPagingContext, BaseAddress, &PageAttribute);
 662     if (PageEntry == NULL) {
 663       Status = RETURN_UNSUPPORTED;
 664       goto Done;
 665     }
 666     PageEntryLength = PageAttributeToLength (PageAttribute);
 667     SplitAttribute = NeedSplitPage (BaseAddress, Length, PageEntry, PageAttribute);
 668     if (SplitAttribute == PageNone) {
 669       ConvertPageEntryAttribute (&CurrentPagingContext, PageEntry, Attributes, PageAction, &IsEntryModified);
 670       if (IsEntryModified) {
 671         if (IsModified != NULL) {
 672           *IsModified = TRUE;
 673         }
 674       }
 675       //
 676       // Convert success, move to next
 677       //
 678       BaseAddress += PageEntryLength;
 679       Length -= PageEntryLength;
 680     } else {
 681       if (AllocatePagesFunc == NULL) {
 682         Status = RETURN_UNSUPPORTED;
 683         goto Done;
 684       }
 685       Status = SplitPage (PageEntry, PageAttribute, SplitAttribute, AllocatePagesFunc);
 686       if (RETURN_ERROR (Status)) {
 687         Status = RETURN_UNSUPPORTED;
 688         goto Done;
 689       }
 690       if (IsSplitted != NULL) {
 691         *IsSplitted = TRUE;
 692       }
 693       if (IsModified != NULL) {
 694         *IsModified = TRUE;
 695       }
 696       //
 697       // Just split current page
 698       // Convert success in next around
 699       //
 700     }
 701   }
 702
 703 Done:
 704   //
 705   // Restore page table write protection, if any.
 706   //
 707   if (IsWpEnabled) {
 708     EnableReadOnlyPageWriteProtect ();
 709   }
 710   return Status;
 711 }
 712
 713 /**
 714   This function assigns the page attributes for the memory region specified by BaseAddress and
 715   Length from their current attributes to the attributes specified by Attributes.
 716
 717   Caller should make sure BaseAddress and Length is at page boundary.
 718
 719   Caller need guarentee the TPL <= TPL_NOTIFY, if there is split page request.
 720
 721   @param[in]  PagingContext     The paging context. NULL means get page table from current CPU context.
 722   @param[in]  BaseAddress       The physical address that is the start address of a memory region.
 723   @param[in]  Length            The size in bytes of the memory region.
 724   @param[in]  Attributes        The bit mask of attributes to set for the memory region.
 725   @param[in]  AllocatePagesFunc If page split is needed, this function is used to allocate more pages.
 726                                 NULL mean page split is unsupported.
 727
 728   @retval RETURN_SUCCESS           The attributes were cleared for the memory region.
 729   @retval RETURN_ACCESS_DENIED     The attributes for the memory resource range specified by
 730                                    BaseAddress and Length cannot be modified.
 731   @retval RETURN_INVALID_PARAMETER Length is zero.
 732                                    Attributes specified an illegal combination of attributes that
 733                                    cannot be set together.
 734   @retval RETURN_OUT_OF_RESOURCES  There are not enough system resources to modify the attributes of
 735                                    the memory resource range.
 736   @retval RETURN_UNSUPPORTED       The processor does not support one or more bytes of the memory
 737                                    resource range specified by BaseAddress and Length.
 738                                    The bit mask of attributes is not support for the memory resource
 739                                    range specified by BaseAddress and Length.
 740 **/
 741 RETURN_STATUS
 742 EFIAPI
 743 AssignMemoryPageAttributes (
 744   IN  PAGE_TABLE_LIB_PAGING_CONTEXT     *PagingContext OPTIONAL,
 745   IN  PHYSICAL_ADDRESS                  BaseAddress,
 746   IN  UINT64                            Length,
 747   IN  UINT64                            Attributes,
 748   IN  PAGE_TABLE_LIB_ALLOCATE_PAGES     AllocatePagesFunc OPTIONAL
 749   )
 750 {
 751   RETURN_STATUS  Status;
 752   BOOLEAN        IsModified;
 753   BOOLEAN        IsSplitted;
 754
 755 //  DEBUG((DEBUG_INFO, "AssignMemoryPageAttributes: 0x%lx - 0x%lx (0x%lx)\n", BaseAddress, Length, Attributes));
 756   Status = ConvertMemoryPageAttributes (PagingContext, BaseAddress, Length, Attributes, PageActionAssign, AllocatePagesFunc, &IsSplitted, &IsModified);
 757   if (!EFI_ERROR(Status)) {
 758     if ((PagingContext == NULL) && IsModified) {
 759       //
 760       // Flush TLB as last step.
 761       //
 762       // Note: Since APs will always init CR3 register in HLT loop mode or do
 763       // TLB flush in MWAIT loop mode, there's no need to flush TLB for them
 764       // here.
 765       //
 766       CpuFlushTlb();
 767     }
 768   }
 769
 770   return Status;
 771 }
 772
 773 /**
 774  Check if Execute Disable feature is enabled or not.
 775 **/
 776 BOOLEAN
 777 IsExecuteDisableEnabled (
 778   VOID
 779   )
 780 {
 781   MSR_CORE_IA32_EFER_REGISTER    MsrEfer;
 782
 783   MsrEfer.Uint64 = AsmReadMsr64 (MSR_IA32_EFER);
 784   return (MsrEfer.Bits.NXE == 1);
 785 }
 786
 787 /**
 788   Update GCD memory space attributes according to current page table setup.
 789 **/
 790 VOID
 791 RefreshGcdMemoryAttributesFromPaging (
 792   VOID
 793   )
 794 {
 795   EFI_STATUS                          Status;
 796   UINTN                               NumberOfDescriptors;
 797   EFI_GCD_MEMORY_SPACE_DESCRIPTOR     *MemorySpaceMap;
 798   PAGE_TABLE_LIB_PAGING_CONTEXT       PagingContext;
 799   PAGE_ATTRIBUTE                      PageAttribute;
 800   UINT64                              *PageEntry;
 801   UINT64                              PageLength;
 802   UINT64                              MemorySpaceLength;
 803   UINT64                              Length;
 804   UINT64                              BaseAddress;
 805   UINT64                              PageStartAddress;
 806   UINT64                              Attributes;
 807   UINT64                              Capabilities;
 808   UINT64                              NewAttributes;
 809   UINTN                               Index;
 810
 811   //
 812   // Assuming that memory space map returned is sorted already; otherwise sort
 813   // them in the order of lowest address to highest address.
 814   //
 815   Status = gDS->GetMemorySpaceMap (&NumberOfDescriptors, &MemorySpaceMap);
 816   ASSERT_EFI_ERROR (Status);
 817
 818   GetCurrentPagingContext (&PagingContext);
 819
 820   Attributes      = 0;
 821   NewAttributes   = 0;
 822   BaseAddress     = 0;
 823   PageLength      = 0;
 824
 825   if (IsExecuteDisableEnabled ()) {
 826     Capabilities = EFI_MEMORY_RO | EFI_MEMORY_RP | EFI_MEMORY_XP;
 827   } else {
 828     Capabilities = EFI_MEMORY_RO | EFI_MEMORY_RP;
 829   }
 830
 831   for (Index = 0; Index < NumberOfDescriptors; Index++) {
 832     if (MemorySpaceMap[Index].GcdMemoryType == EfiGcdMemoryTypeNonExistent) {
 833       continue;
 834     }
 835
 836     //
 837     // Sync the actual paging related capabilities back to GCD service first.
 838     // As a side effect (good one), this can also help to avoid unnecessary
 839     // memory map entries due to the different capabilities of the same type
 840     // memory, such as multiple RT_CODE and RT_DATA entries in memory map,
 841     // which could cause boot failure of some old Linux distro (before v4.3).
 842     //
 843     Status = gDS->SetMemorySpaceCapabilities (
 844                     MemorySpaceMap[Index].BaseAddress,
 845                     MemorySpaceMap[Index].Length,
 846                     MemorySpaceMap[Index].Capabilities | Capabilities
 847                     );
 848     if (EFI_ERROR (Status)) {
 849       //
 850       // If we cannot udpate the capabilities, we cannot update its
 851       // attributes either. So just simply skip current block of memory.
 852       //
 853       DEBUG ((
 854         DEBUG_WARN,
 855         "Failed to update capability: [%lu] %016lx - %016lx (%016lx -> %016lx)\r\n",
 856         (UINT64)Index, MemorySpaceMap[Index].BaseAddress,
 857         MemorySpaceMap[Index].BaseAddress + MemorySpaceMap[Index].Length - 1,
 858         MemorySpaceMap[Index].Capabilities,
 859         MemorySpaceMap[Index].Capabilities | Capabilities
 860         ));
 861       continue;
 862     }
 863
 864     if (MemorySpaceMap[Index].BaseAddress >= (BaseAddress + PageLength)) {
 865       //
 866       // Current memory space starts at a new page. Resetting PageLength will
 867       // trigger a retrieval of page attributes at new address.
 868       //
 869       PageLength = 0;
 870     } else {
 871       //
 872       // In case current memory space is not adjacent to last one
 873       //
 874       PageLength -= (MemorySpaceMap[Index].BaseAddress - BaseAddress);
 875     }
 876
 877     //
 878     // Sync actual page attributes to GCD
 879     //
 880     BaseAddress       = MemorySpaceMap[Index].BaseAddress;
 881     MemorySpaceLength = MemorySpaceMap[Index].Length;
 882     while (MemorySpaceLength > 0) {
 883       if (PageLength == 0) {
 884         PageEntry = GetPageTableEntry (&PagingContext, BaseAddress, &PageAttribute);
 885         if (PageEntry == NULL) {
 886           break;
 887         }
 888
 889         //
 890         // Note current memory space might start in the middle of a page
 891         //
 892         PageStartAddress  = (*PageEntry) & (UINT64)PageAttributeToMask(PageAttribute);
 893         PageLength        = PageAttributeToLength (PageAttribute) - (BaseAddress - PageStartAddress);
 894         Attributes        = GetAttributesFromPageEntry (PageEntry);
 895       }
 896
 897       Length = MIN (PageLength, MemorySpaceLength);
 898       if (Attributes != (MemorySpaceMap[Index].Attributes &
 899                          EFI_MEMORY_PAGETYPE_MASK)) {
 900         NewAttributes = (MemorySpaceMap[Index].Attributes &
 901                          ~EFI_MEMORY_PAGETYPE_MASK) | Attributes;
 902         Status = gDS->SetMemorySpaceAttributes (
 903                         BaseAddress,
 904                         Length,
 905                         NewAttributes
 906                         );
 907         ASSERT_EFI_ERROR (Status);
 908         DEBUG ((
 909           DEBUG_VERBOSE,
 910           "Updated memory space attribute: [%lu] %016lx - %016lx (%016lx -> %016lx)\r\n",
 911           (UINT64)Index, BaseAddress, BaseAddress + Length - 1,
 912           MemorySpaceMap[Index].Attributes,
 913           NewAttributes
 914           ));
 915       }
 916
 917       PageLength        -= Length;
 918       MemorySpaceLength -= Length;
 919       BaseAddress       += Length;
 920     }
 921   }
 922
 923   FreePool (MemorySpaceMap);
 924 }
 925
 926 /**
 927   Initialize a buffer pool for page table use only.
 928
 929   To reduce the potential split operation on page table, the pages reserved for
 930   page table should be allocated in the times of PAGE_TABLE_POOL_UNIT_PAGES and
 931   at the boundary of PAGE_TABLE_POOL_ALIGNMENT. So the page pool is always
 932   initialized with number of pages greater than or equal to the given PoolPages.
 933
 934   Once the pages in the pool are used up, this method should be called again to
 935   reserve at least another PAGE_TABLE_POOL_UNIT_PAGES. Usually this won't happen
 936   often in practice.
 937
 938   @param[in] PoolPages      The least page number of the pool to be created.
 939
 940   @retval TRUE    The pool is initialized successfully.
 941   @retval FALSE   The memory is out of resource.
 942 **/
 943 BOOLEAN
 944 InitializePageTablePool (
 945   IN  UINTN                           PoolPages
 946   )
 947 {
 948   VOID                      *Buffer;
 949   BOOLEAN                   IsModified;
 950
 951   //
 952   // Always reserve at least PAGE_TABLE_POOL_UNIT_PAGES, including one page for
 953   // header.
 954   //
 955   PoolPages += 1;   // Add one page for header.
 956   PoolPages = ((PoolPages - 1) / PAGE_TABLE_POOL_UNIT_PAGES + 1) *
 957               PAGE_TABLE_POOL_UNIT_PAGES;
 958   Buffer = AllocateAlignedPages (PoolPages, PAGE_TABLE_POOL_ALIGNMENT);
 959   if (Buffer == NULL) {
 960     DEBUG ((DEBUG_ERROR, "ERROR: Out of aligned pages\r\n"));
 961     return FALSE;
 962   }
 963
 964   //
 965   // Link all pools into a list for easier track later.
 966   //
 967   if (mPageTablePool == NULL) {
 968     mPageTablePool = Buffer;
 969     mPageTablePool->NextPool = mPageTablePool;
 970   } else {
 971     ((PAGE_TABLE_POOL *)Buffer)->NextPool = mPageTablePool->NextPool;
 972     mPageTablePool->NextPool = Buffer;
 973     mPageTablePool = Buffer;
 974   }
 975
 976   //
 977   // Reserve one page for pool header.
 978   //
 979   mPageTablePool->FreePages  = PoolPages - 1;
 980   mPageTablePool->Offset = EFI_PAGES_TO_SIZE (1);
 981
 982   //
 983   // Mark the whole pool pages as read-only.
 984   //
 985   ConvertMemoryPageAttributes (
 986     NULL,
 987     (PHYSICAL_ADDRESS)(UINTN)Buffer,
 988     EFI_PAGES_TO_SIZE (PoolPages),
 989     EFI_MEMORY_RO,
 990     PageActionSet,
 991     AllocatePageTableMemory,
 992     NULL,
 993     &IsModified
 994     );
 995   ASSERT (IsModified == TRUE);
 996
 997   return TRUE;
 998 }
 999
1000 /**
1001   This API provides a way to allocate memory for page table.
1002
1003   This API can be called more than once to allocate memory for page tables.
1004
1005   Allocates the number of 4KB pages and returns a pointer to the allocated
1006   buffer. The buffer returned is aligned on a 4KB boundary.
1007
1008   If Pages is 0, then NULL is returned.
1009   If there is not enough memory remaining to satisfy the request, then NULL is
1010   returned.
1011
1012   @param  Pages                 The number of 4 KB pages to allocate.
1013
1014   @return A pointer to the allocated buffer or NULL if allocation fails.
1015
1016 **/
1017 VOID *
1018 EFIAPI
1019 AllocatePageTableMemory (
1020   IN UINTN           Pages
1021   )
1022 {
1023   VOID                            *Buffer;
1024
1025   if (Pages == 0) {
1026     return NULL;
1027   }
1028
1029   //
1030   // Renew the pool if necessary.
1031   //
1032   if (mPageTablePool == NULL ||
1033       Pages > mPageTablePool->FreePages) {
1034     if (!InitializePageTablePool (Pages)) {
1035       return NULL;
1036     }
1037   }
1038
1039   Buffer = (UINT8 *)mPageTablePool + mPageTablePool->Offset;
1040
1041   mPageTablePool->Offset     += EFI_PAGES_TO_SIZE (Pages);
1042   mPageTablePool->FreePages  -= Pages;
1043
1044   return Buffer;
1045 }
1046
1047 /**
1048   Initialize the Page Table lib.
1049 **/
1050 VOID
1051 InitializePageTableLib (
1052   VOID
1053   )
1054 {
1055   PAGE_TABLE_LIB_PAGING_CONTEXT     CurrentPagingContext;
1056
1057   GetCurrentPagingContext (&CurrentPagingContext);
1058
1059   //
1060   // Reserve memory of page tables for future uses, if paging is enabled.
1061   //
1062   if (CurrentPagingContext.ContextData.X64.PageTableBase != 0 &&
1063       (CurrentPagingContext.ContextData.Ia32.Attributes &
1064        PAGE_TABLE_LIB_PAGING_CONTEXT_IA32_X64_ATTRIBUTES_PAE) != 0) {
1065     DisableReadOnlyPageWriteProtect ();
1066     InitializePageTablePool (1);
1067     EnableReadOnlyPageWriteProtect ();
1068   }
1069
1070   DEBUG ((DEBUG_INFO, "CurrentPagingContext:\n", CurrentPagingContext.MachineType));
1071   DEBUG ((DEBUG_INFO, "  MachineType   - 0x%x\n", CurrentPagingContext.MachineType));
1072   DEBUG ((DEBUG_INFO, "  PageTableBase - 0x%x\n", CurrentPagingContext.ContextData.X64.PageTableBase));
1073   DEBUG ((DEBUG_INFO, "  Attributes    - 0x%x\n", CurrentPagingContext.ContextData.X64.Attributes));
1074
1075   return ;
1076 }
1077